You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sb...@apache.org on 2019/04/16 09:45:25 UTC
[arrow] branch master updated: ARROW-5108: [Go] implement reading
primitive arrays from Arrow file
This is an automated email from the ASF dual-hosted git repository.
sbinet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 1c0930d ARROW-5108: [Go] implement reading primitive arrays from Arrow file
1c0930d is described below
commit 1c0930dd656b9af3bbe373c1d5f63f1a004672ba
Author: Sebastien Binet <bi...@cern.ch>
AuthorDate: Tue Apr 16 11:45:10 2019 +0200
ARROW-5108: [Go] implement reading primitive arrays from Arrow file
This CL adds the generation of flatbuffers-based files for the IPC backend.
This CL adds the basic functionality to read an Arrow file.
Author: Sebastien Binet <bi...@cern.ch>
Closes #4114 from sbinet/issue-5108 and squashes the following commits:
ede1b345 <Sebastien Binet> arrow/ipc: address comment about double (de)serializing
200a01ad <Sebastien Binet> arrow/ipc/cmd/arrow-{cat,ls}: refactor to ease testing
f523d065 <Sebastien Binet> arrow/ipc: store bitwidth
ee6ab29a <Sebastien Binet> arrow/ipc: implement PR reviews for NewMessage
b87c051f <Sebastien Binet> arrow/ipc: implement PR reviews
00f70e0a <Sebastien Binet> arrow/ipc: prepare for file writer and stream writer
ea7efa79 <Sebastien Binet> arrow/ipc: cosmetics
92aa9684 <Sebastien Binet> ci: disable -race on windows
fa979e7f <Sebastien Binet> arrow/ipc/cmd/arrow-cat: detect stream/file and be resilient
0e7392ca <Sebastien Binet> ci: fixups for Go commands
7891d45c <Sebastien Binet> arrow/ipc/cmd/arrow-{cat,ls}: first import
e28f7105 <Sebastien Binet> arrow/ipc: impl read streamer
55f32510 <Sebastien Binet> arrow/ipc: first stabs at Reader
5380ee56 <Sebastien Binet> arrow/ipc: more dictMemo tests
0b2ed485 <Sebastien Binet> arrow/ipc: add more dictMemo tests
d9ea22a6 <Sebastien Binet> arrow/array: test Boolean.Stringer
4dfcd784 <Sebastien Binet> arrow/internal/bitutil: add tests for IsMultipleOf8
011f7c65 <Sebastien Binet> arrow/ipc: rename Message.ref into refCount
080fb239 <Sebastien Binet> go.mod
40a9927c <Sebastien Binet> go.sum: temporarily disable go.sum for flatbuffers
1f4cf5c8 <Sebastien Binet> ARROW-5108: implement reading primitive arrays from Arrow file
---
appveyor.yml | 7 +-
ci/go-build-main.bat | 2 +-
go/arrow/array/boolean_test.go | 28 ++
go/arrow/doc.go | 1 +
go/arrow/gen-flatbuffers.go | 155 +++++++
go/arrow/go.mod | 1 +
go/arrow/internal/bitutil/bitutil.go | 3 +
go/arrow/internal/bitutil/bitutil_test.go | 30 ++
go/arrow/{go.mod => internal/flatbuf/Binary.go} | 38 +-
go/arrow/internal/flatbuf/Block.go | 74 ++++
go/arrow/{go.mod => internal/flatbuf/Bool.go} | 38 +-
go/arrow/internal/flatbuf/Buffer.go | 67 +++
go/arrow/internal/flatbuf/Date.go | 71 ++++
go/arrow/{go.mod => internal/flatbuf/DateUnit.go} | 19 +-
go/arrow/internal/flatbuf/Decimal.go | 84 ++++
go/arrow/internal/flatbuf/DictionaryBatch.go | 106 +++++
go/arrow/internal/flatbuf/DictionaryEncoding.go | 116 ++++++
.../{go.mod => internal/flatbuf/Endianness.go} | 21 +-
go/arrow/internal/flatbuf/Field.go | 178 ++++++++
go/arrow/internal/flatbuf/FieldNode.go | 76 ++++
go/arrow/internal/flatbuf/FixedSizeBinary.go | 67 +++
go/arrow/internal/flatbuf/FixedSizeList.go | 67 +++
go/arrow/internal/flatbuf/FloatingPoint.go | 65 +++
go/arrow/internal/flatbuf/Footer.go | 134 ++++++
go/arrow/internal/flatbuf/Int.go | 80 ++++
go/arrow/internal/flatbuf/Interval.go | 65 +++
.../{go.mod => internal/flatbuf/IntervalUnit.go} | 19 +-
go/arrow/internal/flatbuf/KeyValue.go | 75 ++++
go/arrow/{go.mod => internal/flatbuf/List.go} | 38 +-
go/arrow/internal/flatbuf/Map.go | 91 +++++
go/arrow/internal/flatbuf/Message.go | 133 ++++++
go/arrow/internal/flatbuf/MessageHeader.go | 47 +++
.../flatbuf/MetadataVersion.go} | 27 +-
go/arrow/{go.mod => internal/flatbuf/Null.go} | 39 +-
go/arrow/{go.mod => internal/flatbuf/Precision.go} | 21 +-
go/arrow/internal/flatbuf/RecordBatch.go | 136 +++++++
go/arrow/internal/flatbuf/Schema.go | 125 ++++++
go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go | 145 +++++++
go/arrow/internal/flatbuf/SparseTensor.go | 175 ++++++++
.../flatbuf/SparseTensorIndex.go} | 21 +-
go/arrow/internal/flatbuf/SparseTensorIndexCOO.go | 115 ++++++
go/arrow/internal/flatbuf/Struct_.go | 53 +++
go/arrow/internal/flatbuf/Tensor.go | 152 +++++++
go/arrow/internal/flatbuf/TensorDim.go | 83 ++++
go/arrow/internal/flatbuf/Time.go | 83 ++++
go/arrow/{go.mod => internal/flatbuf/TimeUnit.go} | 23 +-
go/arrow/internal/flatbuf/Timestamp.go | 122 ++++++
go/arrow/internal/flatbuf/Type.go | 66 +++
go/arrow/internal/flatbuf/Union.go | 92 +++++
go/arrow/{go.mod => internal/flatbuf/UnionMode.go} | 19 +-
go/arrow/{go.mod => internal/flatbuf/Utf8.go} | 39 +-
go/arrow/ipc/cmd/arrow-cat/main.go | 210 ++++++++++
go/arrow/ipc/cmd/arrow-ls/main.go | 205 ++++++++++
go/arrow/ipc/dict.go | 85 ++++
go/arrow/ipc/dict_test.go | 196 +++++++++
go/arrow/ipc/feather/fbs/CTable.go | 134 ++++++
go/arrow/ipc/feather/fbs/CategoryMetadata.go | 85 ++++
go/arrow/ipc/feather/fbs/Column.go | 117 ++++++
.../{go.mod => ipc/feather/fbs/DateMetadata.go} | 38 +-
go/arrow/{go.mod => ipc/feather/fbs/Encoding.go} | 25 +-
go/arrow/ipc/feather/fbs/PrimitiveArray.go | 150 +++++++
go/arrow/ipc/feather/fbs/TimeMetadata.go | 65 +++
go/arrow/{go.mod => ipc/feather/fbs/TimeUnit.go} | 23 +-
go/arrow/ipc/feather/fbs/TimestampMetadata.go | 80 ++++
go/arrow/ipc/feather/fbs/Type.go | 67 +++
.../{go.mod => ipc/feather/fbs/TypeMetadata.go} | 25 +-
go/arrow/ipc/file_reader.go | 453 +++++++++++++++++++++
go/arrow/ipc/ipc.go | 71 ++++
go/arrow/ipc/message.go | 215 ++++++++++
go/arrow/ipc/metadata.go | 412 +++++++++++++++++++
go/arrow/ipc/reader.go | 179 ++++++++
go/arrow/schema.go | 25 ++
go/arrow/schema_test.go | 28 ++
73 files changed, 6315 insertions(+), 105 deletions(-)
diff --git a/appveyor.yml b/appveyor.yml
index 7c83b7e..afc2649 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -34,6 +34,7 @@ only_commits:
cache:
- C:\Users\Appveyor\clcache
- C:\Users\Appveyor\.cargo\registry
+ - '%LocalAppData%\\go-build'
matrix:
fast_finish: true
@@ -79,13 +80,15 @@ environment:
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
USE_CLCACHE: false
- JOB: "Go"
- TARGET: x86_64-pc-windows-msvc
+ MINGW_PACKAGE_PREFIX: mingw-w64-x86_64
+ MINGW_PREFIX: c:\msys64\mingw64
+ MSYSTEM: MINGW64
+ USE_CLCACHE: false
GOROOT: 'c:\go111'
GOPATH: c:\gopath
GO111MODULE: on
GOTOOLDIR: '%GOROOT%\pkg\tool\windows_amd64'
PATH: '%GOPATH%\bin;%GOROOT%\bin;%PATH%'
- USE_CLCACHE: false
MSVC_DEFAULT_OPTIONS: ON
APPVEYOR_SAVE_CACHE_ON_ERROR: true
diff --git a/ci/go-build-main.bat b/ci/go-build-main.bat
index 4b4dc63..9c29808 100644
--- a/ci/go-build-main.bat
+++ b/ci/go-build-main.bat
@@ -21,6 +21,6 @@ pushd go\arrow
%GOROOT%\bin\go version
%GOROOT%\bin\go get -v ./... || exit /B
-%GOROOT%\bin\go test -race ./... || exit /B
+%GOROOT%\bin\go test ./... || exit /B
popd
diff --git a/go/arrow/array/boolean_test.go b/go/arrow/array/boolean_test.go
index e6f4b9b..e2d5805 100644
--- a/go/arrow/array/boolean_test.go
+++ b/go/arrow/array/boolean_test.go
@@ -17,7 +17,9 @@
package array_test
import (
+ "fmt"
"reflect"
+ "strings"
"testing"
"github.com/apache/arrow/go/arrow/array"
@@ -258,3 +260,29 @@ func TestBooleanSliceOutOfBounds(t *testing.T) {
})
}
}
+
+func TestBooleanStringer(t *testing.T) {
+ pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer pool.AssertSize(t, 0)
+
+ var (
+ values = []bool{true, false, true, false, true, false, true, false, true, false}
+ valids = []bool{true, true, false, true, true, true, false, true, true, true}
+ )
+
+ b := array.NewBooleanBuilder(pool)
+ defer b.Release()
+
+ b.AppendValues(values, valids)
+
+ arr := b.NewArray().(*array.Boolean)
+ defer arr.Release()
+
+ out := new(strings.Builder)
+ fmt.Fprintf(out, "%v", arr)
+
+ const want = "[true false (null) false true false (null) false true false]"
+ if got := out.String(); got != want {
+ t.Fatalf("invalid stringer:\ngot= %q\nwant=%q", got, want)
+ }
+}
diff --git a/go/arrow/doc.go b/go/arrow/doc.go
index ebd9c8e..a91e62c 100644
--- a/go/arrow/doc.go
+++ b/go/arrow/doc.go
@@ -33,6 +33,7 @@ package arrow
//go:generate go run _tools/tmpl/main.go -i -data=numeric.tmpldata type_traits_numeric.gen.go.tmpl array/numeric.gen.go.tmpl array/numericbuilder.gen.go.tmpl array/bufferbuilder_numeric.gen.go.tmpl
//go:generate go run _tools/tmpl/main.go -i -data=datatype_numeric.gen.go.tmpldata datatype_numeric.gen.go.tmpl tensor/numeric.gen.go.tmpl tensor/numeric.gen_test.go.tmpl
+//go:generate go run ./gen-flatbuffers.go
// stringer
//go:generate stringer -type=Type
diff --git a/go/arrow/gen-flatbuffers.go b/go/arrow/gen-flatbuffers.go
new file mode 100644
index 0000000..265d49f
--- /dev/null
+++ b/go/arrow/gen-flatbuffers.go
@@ -0,0 +1,155 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build ignore
+
+package main
+
+import (
+ "bytes"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+)
+
+func main() {
+ dir, err := ioutil.TempDir("", "go-arrow-")
+ if err != nil {
+ log.Fatalf("could not create top-level temporary directory: %v", err)
+ }
+ defer os.RemoveAll(dir)
+
+ genFormat(dir)
+ genFeather(dir)
+}
+
+func genFormat(dir string) {
+ args := []string{"--go", "-o", filepath.Join(dir, "format")}
+ fnames, err := filepath.Glob("../../format/*.fbs")
+ if err != nil || len(fnames) == 0 {
+ log.Fatalf("could not retrieve list of format FlatBuffers files: files=%d err=%v",
+ len(fnames), err,
+ )
+ }
+ args = append(args, fnames...)
+
+ gen := exec.Command("flatc", args...)
+ gen.Stdout = os.Stdout
+ gen.Stderr = os.Stderr
+
+ err = gen.Run()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ err = os.MkdirAll("./internal/flatbuf", 0755)
+ if err != nil {
+ log.Fatalf("could not create ./internal/flatbuf directory: %v", err)
+ }
+
+ base := filepath.Join(dir, "format", "org", "apache", "arrow", "flatbuf")
+ fnames, err = filepath.Glob(filepath.Join(base, "*.go"))
+ if err != nil {
+ log.Fatalf("could not glob %v/*.go: %v", base, err)
+ }
+
+ for _, fname := range fnames {
+ dst := filepath.Join(".", "internal", "flatbuf", filepath.Base(fname))
+ process(dst, fname)
+ }
+}
+
+func genFeather(dir string) {
+ gen := exec.Command("flatc",
+ "--go", "-o", filepath.Join(dir, "feather"),
+ "../../cpp/src/arrow/ipc/feather.fbs",
+ )
+
+ gen.Stdout = os.Stdout
+ gen.Stderr = os.Stderr
+
+ err := gen.Run()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ err = os.MkdirAll("./ipc/feather/fbs", 0755)
+ if err != nil {
+ log.Fatalf("could not create ./feather/fbs directory: %v", err)
+ }
+
+ base := filepath.Join(dir, "feather", "arrow")
+ fnames, err := filepath.Glob(filepath.Join(base, "ipc", "feather", "*", "*.go"))
+ if err != nil {
+ log.Fatalf("could not glob ipc/feather/*.go: %v", err)
+ }
+
+ for _, fname := range fnames {
+ dst := strings.Replace(fname, base, ".", -1)
+ process(dst, fname)
+ }
+}
+
+func process(dst, fname string) {
+ raw, err := ioutil.ReadFile(fname)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ f, err := os.Create(dst)
+ if err != nil {
+ log.Fatal(err)
+ }
+ defer f.Close()
+
+ if !bytes.HasPrefix(raw, []byte(hdr)) {
+ _, err = f.Write([]byte(hdr))
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+
+ _, err = f.Write(raw)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ err = f.Close()
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+const hdr = `// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+`
diff --git a/go/arrow/go.mod b/go/arrow/go.mod
index 451f68e..ac0e6d5 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/go.mod
@@ -18,6 +18,7 @@ module github.com/apache/arrow/go/arrow
require (
github.com/davecgh/go-spew v1.1.0 // indirect
+ github.com/google/flatbuffers v1.10.0
github.com/pkg/errors v0.8.1
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.2.0
diff --git a/go/arrow/internal/bitutil/bitutil.go b/go/arrow/internal/bitutil/bitutil.go
index ef25a99..06a2a95 100644
--- a/go/arrow/internal/bitutil/bitutil.go
+++ b/go/arrow/internal/bitutil/bitutil.go
@@ -27,6 +27,9 @@ var (
FlippedBitMask = [8]byte{254, 253, 251, 247, 239, 223, 191, 127}
)
+// IsMultipleOf8 returns whether v is a multiple of 8.
+func IsMultipleOf8(v int64) bool { return v&7 == 0 }
+
// NextPowerOf2 rounds x to the next power of two.
func NextPowerOf2(x int) int { return 1 << uint(bits.Len(uint(x))) }
diff --git a/go/arrow/internal/bitutil/bitutil_test.go b/go/arrow/internal/bitutil/bitutil_test.go
index 108db9a..62134dc 100644
--- a/go/arrow/internal/bitutil/bitutil_test.go
+++ b/go/arrow/internal/bitutil/bitutil_test.go
@@ -17,6 +17,7 @@
package bitutil_test
import (
+ "fmt"
"math/rand"
"testing"
@@ -25,6 +26,35 @@ import (
"github.com/stretchr/testify/assert"
)
+func TestIsMultipleOf8(t *testing.T) {
+ for _, tc := range []struct {
+ v int64
+ want bool
+ }{
+ {-16, true},
+ {-9, false},
+ {-8, true},
+ {-7, false},
+ {-4, false},
+ {-1, false},
+ {-0, true},
+ {0, true},
+ {1, false},
+ {4, false},
+ {7, false},
+ {8, true},
+ {9, false},
+ {16, true},
+ } {
+ t.Run(fmt.Sprintf("v=%d", tc.v), func(t *testing.T) {
+ got := bitutil.IsMultipleOf8(tc.v)
+ if got != tc.want {
+ t.Fatalf("IsMultipleOf8(%d): got=%v, want=%v", tc.v, got, tc.want)
+ }
+ })
+ }
+}
+
func TestCeilByte(t *testing.T) {
tests := []struct {
name string
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/Binary.go
similarity index 53%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/Binary.go
index 451f68e..25b617a 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/Binary.go
@@ -14,11 +14,37 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
)
+
+type Binary struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsBinary(buf []byte, offset flatbuffers.UOffsetT) *Binary {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Binary{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Binary) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Binary) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func BinaryStart(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func BinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Block.go b/go/arrow/internal/flatbuf/Block.go
new file mode 100644
index 0000000..9172778
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Block.go
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Block struct {
+ _tab flatbuffers.Struct
+}
+
+func (rcv *Block) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Block) Table() flatbuffers.Table {
+ return rcv._tab.Table
+}
+
+/// Index to the start of the RecordBlock (note this is past the Message header)
+func (rcv *Block) Offset() int64 {
+ return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
+}
+/// Index to the start of the RecordBlock (note this is past the Message header)
+func (rcv *Block) MutateOffset(n int64) bool {
+ return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
+}
+
+/// Length of the metadata
+func (rcv *Block) MetaDataLength() int32 {
+ return rcv._tab.GetInt32(rcv._tab.Pos + flatbuffers.UOffsetT(8))
+}
+/// Length of the metadata
+func (rcv *Block) MutateMetaDataLength(n int32) bool {
+ return rcv._tab.MutateInt32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
+}
+
+/// Length of the data (this is aligned so there can be a gap between this and
+/// the metatdata).
+func (rcv *Block) BodyLength() int64 {
+ return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(16))
+}
+/// Length of the data (this is aligned so there can be a gap between this and
+/// the metatdata).
+func (rcv *Block) MutateBodyLength(n int64) bool {
+ return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n)
+}
+
+func CreateBlock(builder *flatbuffers.Builder, offset int64, metaDataLength int32, bodyLength int64) flatbuffers.UOffsetT {
+ builder.Prep(8, 24)
+ builder.PrependInt64(bodyLength)
+ builder.Pad(4)
+ builder.PrependInt32(metaDataLength)
+ builder.PrependInt64(offset)
+ return builder.Offset()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/Bool.go
similarity index 54%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/Bool.go
index 451f68e..6a4a9d2 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/Bool.go
@@ -14,11 +14,37 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
)
+
+type Bool struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsBool(buf []byte, offset flatbuffers.UOffsetT) *Bool {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Bool{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Bool) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Bool) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func BoolStart(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func BoolEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Buffer.go b/go/arrow/internal/flatbuf/Buffer.go
new file mode 100644
index 0000000..171366d
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Buffer.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// A Buffer represents a single contiguous memory segment
+type Buffer struct {
+ _tab flatbuffers.Struct
+}
+
+func (rcv *Buffer) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Buffer) Table() flatbuffers.Table {
+ return rcv._tab.Table
+}
+
+/// The relative offset into the shared memory page where the bytes for this
+/// buffer starts
+func (rcv *Buffer) Offset() int64 {
+ return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
+}
+/// The relative offset into the shared memory page where the bytes for this
+/// buffer starts
+func (rcv *Buffer) MutateOffset(n int64) bool {
+ return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
+}
+
+/// The absolute length (in bytes) of the memory buffer. The memory is found
+/// from offset (inclusive) to offset + length (non-inclusive).
+func (rcv *Buffer) Length() int64 {
+ return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8))
+}
+/// The absolute length (in bytes) of the memory buffer. The memory is found
+/// from offset (inclusive) to offset + length (non-inclusive).
+func (rcv *Buffer) MutateLength(n int64) bool {
+ return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
+}
+
+func CreateBuffer(builder *flatbuffers.Builder, offset int64, length int64) flatbuffers.UOffsetT {
+ builder.Prep(8, 16)
+ builder.PrependInt64(length)
+ builder.PrependInt64(offset)
+ return builder.Offset()
+}
diff --git a/go/arrow/internal/flatbuf/Date.go b/go/arrow/internal/flatbuf/Date.go
new file mode 100644
index 0000000..d1fce94
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Date.go
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX
+/// epoch (1970-01-01), stored in either of two units:
+///
+/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no
+/// leap seconds), where the values are evenly divisible by 86400000
+/// * Days (32 bits) since the UNIX epoch
+type Date struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsDate(buf []byte, offset flatbuffers.UOffsetT) *Date {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Date{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Date) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Date) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Date) Unit() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 1
+}
+
+func (rcv *Date) MutateUnit(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func DateStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func DateAddUnit(builder *flatbuffers.Builder, unit int16) {
+ builder.PrependInt16Slot(0, unit, 1)
+}
+func DateEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/DateUnit.go
similarity index 74%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/DateUnit.go
index 451f68e..bb49f28 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/DateUnit.go
@@ -14,11 +14,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type DateUnit = int16
+const (
+ DateUnitDAY DateUnit = 0
+ DateUnitMILLISECOND DateUnit = 1
)
+
+var EnumNamesDateUnit = map[DateUnit]string{
+ DateUnitDAY:"DAY",
+ DateUnitMILLISECOND:"MILLISECOND",
+}
+
diff --git a/go/arrow/internal/flatbuf/Decimal.go b/go/arrow/internal/flatbuf/Decimal.go
new file mode 100644
index 0000000..a0a9b50
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Decimal.go
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Decimal struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsDecimal(buf []byte, offset flatbuffers.UOffsetT) *Decimal {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Decimal{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Decimal) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Decimal) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// Total number of decimal digits
+func (rcv *Decimal) Precision() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Total number of decimal digits
+func (rcv *Decimal) MutatePrecision(n int32) bool {
+ return rcv._tab.MutateInt32Slot(4, n)
+}
+
+/// Number of digits after the decimal point "."
+func (rcv *Decimal) Scale() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Number of digits after the decimal point "."
+func (rcv *Decimal) MutateScale(n int32) bool {
+ return rcv._tab.MutateInt32Slot(6, n)
+}
+
+func DecimalStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func DecimalAddPrecision(builder *flatbuffers.Builder, precision int32) {
+ builder.PrependInt32Slot(0, precision, 0)
+}
+func DecimalAddScale(builder *flatbuffers.Builder, scale int32) {
+ builder.PrependInt32Slot(1, scale, 0)
+}
+func DecimalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/DictionaryBatch.go b/go/arrow/internal/flatbuf/DictionaryBatch.go
new file mode 100644
index 0000000..ad3bc17
--- /dev/null
+++ b/go/arrow/internal/flatbuf/DictionaryBatch.go
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// For sending dictionary encoding information. Any Field can be
+/// dictionary-encoded, but in this case none of its children may be
+/// dictionary-encoded.
+/// There is one vector / column per dictionary, but that vector / column
+/// may be spread across multiple dictionary batches by using the isDelta
+/// flag
+type DictionaryBatch struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsDictionaryBatch(buf []byte, offset flatbuffers.UOffsetT) *DictionaryBatch {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &DictionaryBatch{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *DictionaryBatch) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *DictionaryBatch) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *DictionaryBatch) Id() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *DictionaryBatch) MutateId(n int64) bool {
+ return rcv._tab.MutateInt64Slot(4, n)
+}
+
+func (rcv *DictionaryBatch) Data(obj *RecordBatch) *RecordBatch {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := rcv._tab.Indirect(o + rcv._tab.Pos)
+ if obj == nil {
+ obj = new(RecordBatch)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// If isDelta is true the values in the dictionary are to be appended to a
+/// dictionary with the indicated id
+func (rcv *DictionaryBatch) IsDelta() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// If isDelta is true the values in the dictionary are to be appended to a
+/// dictionary with the indicated id
+func (rcv *DictionaryBatch) MutateIsDelta(n byte) bool {
+ return rcv._tab.MutateByteSlot(8, n)
+}
+
+func DictionaryBatchStart(builder *flatbuffers.Builder) {
+ builder.StartObject(3)
+}
+func DictionaryBatchAddId(builder *flatbuffers.Builder, id int64) {
+ builder.PrependInt64Slot(0, id, 0)
+}
+func DictionaryBatchAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(data), 0)
+}
+func DictionaryBatchAddIsDelta(builder *flatbuffers.Builder, isDelta byte) {
+ builder.PrependByteSlot(2, isDelta, 0)
+}
+func DictionaryBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/DictionaryEncoding.go b/go/arrow/internal/flatbuf/DictionaryEncoding.go
new file mode 100644
index 0000000..1e7c12f
--- /dev/null
+++ b/go/arrow/internal/flatbuf/DictionaryEncoding.go
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// Dictionary encoding metadata
+type DictionaryEncoding struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsDictionaryEncoding(buf []byte, offset flatbuffers.UOffsetT) *DictionaryEncoding {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &DictionaryEncoding{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *DictionaryEncoding) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *DictionaryEncoding) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// The known dictionary id in the application where this data is used. In
+/// the file or streaming formats, the dictionary ids are found in the
+/// DictionaryBatch messages
+func (rcv *DictionaryEncoding) Id() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// The known dictionary id in the application where this data is used. In
+/// the file or streaming formats, the dictionary ids are found in the
+/// DictionaryBatch messages
+func (rcv *DictionaryEncoding) MutateId(n int64) bool {
+ return rcv._tab.MutateInt64Slot(4, n)
+}
+
+/// The dictionary indices are constrained to be positive integers. If this
+/// field is null, the indices must be signed int32
+func (rcv *DictionaryEncoding) IndexType(obj *Int) *Int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := rcv._tab.Indirect(o + rcv._tab.Pos)
+ if obj == nil {
+ obj = new(Int)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// The dictionary indices are constrained to be positive integers. If this
+/// field is null, the indices must be signed int32
+/// By default, dictionaries are not ordered, or the order does not have
+/// semantic meaning. In some statistical, applications, dictionary-encoding
+/// is used to represent ordered categorical data, and we provide a way to
+/// preserve that metadata here
+func (rcv *DictionaryEncoding) IsOrdered() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// By default, dictionaries are not ordered, or the order does not have
+/// semantic meaning. In some statistical, applications, dictionary-encoding
+/// is used to represent ordered categorical data, and we provide a way to
+/// preserve that metadata here
+func (rcv *DictionaryEncoding) MutateIsOrdered(n byte) bool {
+ return rcv._tab.MutateByteSlot(8, n)
+}
+
+func DictionaryEncodingStart(builder *flatbuffers.Builder) {
+ builder.StartObject(3)
+}
+func DictionaryEncodingAddId(builder *flatbuffers.Builder, id int64) {
+ builder.PrependInt64Slot(0, id, 0)
+}
+func DictionaryEncodingAddIndexType(builder *flatbuffers.Builder, indexType flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(indexType), 0)
+}
+func DictionaryEncodingAddIsOrdered(builder *flatbuffers.Builder, isOrdered byte) {
+ builder.PrependByteSlot(2, isOrdered, 0)
+}
+func DictionaryEncodingEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/Endianness.go
similarity index 66%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/Endianness.go
index 451f68e..22e349e 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/Endianness.go
@@ -14,11 +14,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+/// ----------------------------------------------------------------------
+/// Endianness of the platform producing the data
+type Endianness = int16
+const (
+ EndiannessLittle Endianness = 0
+ EndiannessBig Endianness = 1
)
+
+var EnumNamesEndianness = map[Endianness]string{
+ EndiannessLittle:"Little",
+ EndiannessBig:"Big",
+}
+
diff --git a/go/arrow/internal/flatbuf/Field.go b/go/arrow/internal/flatbuf/Field.go
new file mode 100644
index 0000000..1310b9d
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Field.go
@@ -0,0 +1,178 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// A field represents a named column in a record / row batch or child of a
+/// nested type.
+///
+/// - children is only for nested Arrow arrays
+/// - For primitive types, children will have length 0
+/// - nullable should default to true in general
+type Field struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsField(buf []byte, offset flatbuffers.UOffsetT) *Field {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Field{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Field) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Field) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Field) Name() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+func (rcv *Field) Nullable() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Field) MutateNullable(n byte) bool {
+ return rcv._tab.MutateByteSlot(6, n)
+}
+
+func (rcv *Field) TypeType() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Field) MutateTypeType(n byte) bool {
+ return rcv._tab.MutateByteSlot(8, n)
+}
+
+func (rcv *Field) Type(obj *flatbuffers.Table) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ rcv._tab.Union(obj, o)
+ return true
+ }
+ return false
+}
+
+func (rcv *Field) Dictionary(obj *DictionaryEncoding) *DictionaryEncoding {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ x := rcv._tab.Indirect(o + rcv._tab.Pos)
+ if obj == nil {
+ obj = new(DictionaryEncoding)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+func (rcv *Field) Children(obj *Field, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Field) ChildrenLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func (rcv *Field) CustomMetadata(obj *KeyValue, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Field) CustomMetadataLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func FieldStart(builder *flatbuffers.Builder) {
+ builder.StartObject(7)
+}
+func FieldAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0)
+}
+func FieldAddNullable(builder *flatbuffers.Builder, nullable byte) {
+ builder.PrependByteSlot(1, nullable, 0)
+}
+func FieldAddTypeType(builder *flatbuffers.Builder, typeType byte) {
+ builder.PrependByteSlot(2, typeType, 0)
+}
+func FieldAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(type_), 0)
+}
+func FieldAddDictionary(builder *flatbuffers.Builder, dictionary flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(dictionary), 0)
+}
+func FieldAddChildren(builder *flatbuffers.Builder, children flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(children), 0)
+}
+func FieldStartChildrenVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func FieldAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(customMetadata), 0)
+}
+func FieldStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func FieldEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/FieldNode.go b/go/arrow/internal/flatbuf/FieldNode.go
new file mode 100644
index 0000000..b8d7115
--- /dev/null
+++ b/go/arrow/internal/flatbuf/FieldNode.go
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// Data structures for describing a table row batch (a collection of
+/// equal-length Arrow arrays)
+/// Metadata about a field at some level of a nested type tree (but not
+/// its children).
+///
+/// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
+/// would have {length: 5, null_count: 2} for its List node, and {length: 6,
+/// null_count: 0} for its Int16 node, as separate FieldNode structs
+type FieldNode struct {
+ _tab flatbuffers.Struct
+}
+
+func (rcv *FieldNode) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *FieldNode) Table() flatbuffers.Table {
+ return rcv._tab.Table
+}
+
+/// The number of value slots in the Arrow array at this level of a nested
+/// tree
+func (rcv *FieldNode) Length() int64 {
+ return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0))
+}
+/// The number of value slots in the Arrow array at this level of a nested
+/// tree
+func (rcv *FieldNode) MutateLength(n int64) bool {
+ return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
+}
+
+/// The number of observed nulls. Fields with null_count == 0 may choose not
+/// to write their physical validity bitmap out as a materialized buffer,
+/// instead setting the length of the bitmap buffer to 0.
+func (rcv *FieldNode) NullCount() int64 {
+ return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8))
+}
+/// The number of observed nulls. Fields with null_count == 0 may choose not
+/// to write their physical validity bitmap out as a materialized buffer,
+/// instead setting the length of the bitmap buffer to 0.
+func (rcv *FieldNode) MutateNullCount(n int64) bool {
+ return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
+}
+
+func CreateFieldNode(builder *flatbuffers.Builder, length int64, nullCount int64) flatbuffers.UOffsetT {
+ builder.Prep(8, 16)
+ builder.PrependInt64(nullCount)
+ builder.PrependInt64(length)
+ return builder.Offset()
+}
diff --git a/go/arrow/internal/flatbuf/FixedSizeBinary.go b/go/arrow/internal/flatbuf/FixedSizeBinary.go
new file mode 100644
index 0000000..4e660d5
--- /dev/null
+++ b/go/arrow/internal/flatbuf/FixedSizeBinary.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type FixedSizeBinary struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsFixedSizeBinary(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeBinary {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &FixedSizeBinary{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *FixedSizeBinary) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *FixedSizeBinary) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// Number of bytes per value
+func (rcv *FixedSizeBinary) ByteWidth() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Number of bytes per value
+func (rcv *FixedSizeBinary) MutateByteWidth(n int32) bool {
+ return rcv._tab.MutateInt32Slot(4, n)
+}
+
+func FixedSizeBinaryStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func FixedSizeBinaryAddByteWidth(builder *flatbuffers.Builder, byteWidth int32) {
+ builder.PrependInt32Slot(0, byteWidth, 0)
+}
+func FixedSizeBinaryEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/FixedSizeList.go b/go/arrow/internal/flatbuf/FixedSizeList.go
new file mode 100644
index 0000000..dabf5cc
--- /dev/null
+++ b/go/arrow/internal/flatbuf/FixedSizeList.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type FixedSizeList struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsFixedSizeList(buf []byte, offset flatbuffers.UOffsetT) *FixedSizeList {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &FixedSizeList{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *FixedSizeList) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *FixedSizeList) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// Number of list items per value
+func (rcv *FixedSizeList) ListSize() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Number of list items per value
+func (rcv *FixedSizeList) MutateListSize(n int32) bool {
+ return rcv._tab.MutateInt32Slot(4, n)
+}
+
+func FixedSizeListStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func FixedSizeListAddListSize(builder *flatbuffers.Builder, listSize int32) {
+ builder.PrependInt32Slot(0, listSize, 0)
+}
+func FixedSizeListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/FloatingPoint.go b/go/arrow/internal/flatbuf/FloatingPoint.go
new file mode 100644
index 0000000..d039f34
--- /dev/null
+++ b/go/arrow/internal/flatbuf/FloatingPoint.go
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type FloatingPoint struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsFloatingPoint(buf []byte, offset flatbuffers.UOffsetT) *FloatingPoint {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &FloatingPoint{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *FloatingPoint) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *FloatingPoint) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *FloatingPoint) Precision() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *FloatingPoint) MutatePrecision(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func FloatingPointStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func FloatingPointAddPrecision(builder *flatbuffers.Builder, precision int16) {
+ builder.PrependInt16Slot(0, precision, 0)
+}
+func FloatingPointEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Footer.go b/go/arrow/internal/flatbuf/Footer.go
new file mode 100644
index 0000000..516e960
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Footer.go
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// Arrow File metadata
+///
+type Footer struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsFooter(buf []byte, offset flatbuffers.UOffsetT) *Footer {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Footer{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Footer) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Footer) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Footer) Version() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Footer) MutateVersion(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func (rcv *Footer) Schema(obj *Schema) *Schema {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := rcv._tab.Indirect(o + rcv._tab.Pos)
+ if obj == nil {
+ obj = new(Schema)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+func (rcv *Footer) Dictionaries(obj *Block, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 24
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Footer) DictionariesLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func (rcv *Footer) RecordBatches(obj *Block, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 24
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Footer) RecordBatchesLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func FooterStart(builder *flatbuffers.Builder) {
+ builder.StartObject(4)
+}
+func FooterAddVersion(builder *flatbuffers.Builder, version int16) {
+ builder.PrependInt16Slot(0, version, 0)
+}
+func FooterAddSchema(builder *flatbuffers.Builder, schema flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(schema), 0)
+}
+func FooterAddDictionaries(builder *flatbuffers.Builder, dictionaries flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(dictionaries), 0)
+}
+func FooterStartDictionariesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(24, numElems, 8)
+}
+func FooterAddRecordBatches(builder *flatbuffers.Builder, recordBatches flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(recordBatches), 0)
+}
+func FooterStartRecordBatchesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(24, numElems, 8)
+}
+func FooterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Int.go b/go/arrow/internal/flatbuf/Int.go
new file mode 100644
index 0000000..791f2fe
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Int.go
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Int struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsInt(buf []byte, offset flatbuffers.UOffsetT) *Int {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Int{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Int) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Int) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Int) BitWidth() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Int) MutateBitWidth(n int32) bool {
+ return rcv._tab.MutateInt32Slot(4, n)
+}
+
+func (rcv *Int) IsSigned() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Int) MutateIsSigned(n byte) bool {
+ return rcv._tab.MutateByteSlot(6, n)
+}
+
+func IntStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func IntAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) {
+ builder.PrependInt32Slot(0, bitWidth, 0)
+}
+func IntAddIsSigned(builder *flatbuffers.Builder, isSigned byte) {
+ builder.PrependByteSlot(1, isSigned, 0)
+}
+func IntEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Interval.go b/go/arrow/internal/flatbuf/Interval.go
new file mode 100644
index 0000000..91023dd
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Interval.go
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Interval struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsInterval(buf []byte, offset flatbuffers.UOffsetT) *Interval {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Interval{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Interval) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Interval) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Interval) Unit() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Interval) MutateUnit(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func IntervalStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func IntervalAddUnit(builder *flatbuffers.Builder, unit int16) {
+ builder.PrependInt16Slot(0, unit, 0)
+}
+func IntervalEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/IntervalUnit.go
similarity index 71%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/IntervalUnit.go
index 451f68e..62baea9 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/IntervalUnit.go
@@ -14,11 +14,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type IntervalUnit = int16
+const (
+ IntervalUnitYEAR_MONTH IntervalUnit = 0
+ IntervalUnitDAY_TIME IntervalUnit = 1
)
+
+var EnumNamesIntervalUnit = map[IntervalUnit]string{
+ IntervalUnitYEAR_MONTH:"YEAR_MONTH",
+ IntervalUnitDAY_TIME:"DAY_TIME",
+}
+
diff --git a/go/arrow/internal/flatbuf/KeyValue.go b/go/arrow/internal/flatbuf/KeyValue.go
new file mode 100644
index 0000000..c1b8531
--- /dev/null
+++ b/go/arrow/internal/flatbuf/KeyValue.go
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// user defined key value pairs to add custom metadata to arrow
+/// key namespacing is the responsibility of the user
+type KeyValue struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsKeyValue(buf []byte, offset flatbuffers.UOffsetT) *KeyValue {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &KeyValue{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *KeyValue) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *KeyValue) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *KeyValue) Key() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+func (rcv *KeyValue) Value() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+func KeyValueStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func KeyValueAddKey(builder *flatbuffers.Builder, key flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(key), 0)
+}
+func KeyValueAddValue(builder *flatbuffers.Builder, value flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(value), 0)
+}
+func KeyValueEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/List.go
similarity index 54%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/List.go
index 451f68e..ba84319 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/List.go
@@ -14,11 +14,37 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
)
+
+type List struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsList(buf []byte, offset flatbuffers.UOffsetT) *List {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &List{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *List) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *List) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func ListStart(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func ListEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Map.go b/go/arrow/internal/flatbuf/Map.go
new file mode 100644
index 0000000..e41f1e3
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Map.go
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// A Map is a logical nested type that is represented as
+///
+/// List<entry: Struct<key: K, value: V>>
+///
+/// In this layout, the keys and values are each respectively contiguous. We do
+/// not constrain the key and value types, so the application is responsible
+/// for ensuring that the keys are hashable and unique. Whether the keys are sorted
+/// may be set in the metadata for this field
+///
+/// In a Field with Map type, the Field has a child Struct field, which then
+/// has two children: key type and the second the value type. The names of the
+/// child fields may be respectively "entry", "key", and "value", but this is
+/// not enforced
+///
+/// Map
+/// - child[0] entry: Struct
+/// - child[0] key: K
+/// - child[1] value: V
+///
+/// Neither the "entry" field nor the "key" field may be nullable.
+///
+/// The metadata is structured so that Arrow systems without special handling
+/// for Map can make Map an alias for List. The "layout" attribute for the Map
+/// field must have the same contents as a List.
+type Map struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsMap(buf []byte, offset flatbuffers.UOffsetT) *Map {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Map{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Map) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Map) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// Set to true if the keys within each value are sorted
+func (rcv *Map) KeysSorted() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Set to true if the keys within each value are sorted
+func (rcv *Map) MutateKeysSorted(n byte) bool {
+ return rcv._tab.MutateByteSlot(4, n)
+}
+
+func MapStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func MapAddKeysSorted(builder *flatbuffers.Builder, keysSorted byte) {
+ builder.PrependByteSlot(0, keysSorted, 0)
+}
+func MapEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Message.go b/go/arrow/internal/flatbuf/Message.go
new file mode 100644
index 0000000..f91ebf6
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Message.go
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Message struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsMessage(buf []byte, offset flatbuffers.UOffsetT) *Message {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Message{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Message) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Message) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Message) Version() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Message) MutateVersion(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func (rcv *Message) HeaderType() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Message) MutateHeaderType(n byte) bool {
+ return rcv._tab.MutateByteSlot(6, n)
+}
+
+func (rcv *Message) Header(obj *flatbuffers.Table) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ rcv._tab.Union(obj, o)
+ return true
+ }
+ return false
+}
+
+func (rcv *Message) BodyLength() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Message) MutateBodyLength(n int64) bool {
+ return rcv._tab.MutateInt64Slot(10, n)
+}
+
+func (rcv *Message) CustomMetadata(obj *KeyValue, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Message) CustomMetadataLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func MessageStart(builder *flatbuffers.Builder) {
+ builder.StartObject(5)
+}
+func MessageAddVersion(builder *flatbuffers.Builder, version int16) {
+ builder.PrependInt16Slot(0, version, 0)
+}
+func MessageAddHeaderType(builder *flatbuffers.Builder, headerType byte) {
+ builder.PrependByteSlot(1, headerType, 0)
+}
+func MessageAddHeader(builder *flatbuffers.Builder, header flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(header), 0)
+}
+func MessageAddBodyLength(builder *flatbuffers.Builder, bodyLength int64) {
+ builder.PrependInt64Slot(3, bodyLength, 0)
+}
+func MessageAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(customMetadata), 0)
+}
+func MessageStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func MessageEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/MessageHeader.go b/go/arrow/internal/flatbuf/MessageHeader.go
new file mode 100644
index 0000000..18730be
--- /dev/null
+++ b/go/arrow/internal/flatbuf/MessageHeader.go
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+/// ----------------------------------------------------------------------
+/// The root Message type
+/// This union enables us to easily send different message types without
+/// redundant storage, and in the future we can easily add new message types.
+///
+/// Arrow implementations do not need to implement all of the message types,
+/// which may include experimental metadata types. For maximum compatibility,
+/// it is best to send data using RecordBatch
+type MessageHeader = byte
+const (
+ MessageHeaderNONE MessageHeader = 0
+ MessageHeaderSchema MessageHeader = 1
+ MessageHeaderDictionaryBatch MessageHeader = 2
+ MessageHeaderRecordBatch MessageHeader = 3
+ MessageHeaderTensor MessageHeader = 4
+ MessageHeaderSparseTensor MessageHeader = 5
+)
+
+var EnumNamesMessageHeader = map[MessageHeader]string{
+ MessageHeaderNONE:"NONE",
+ MessageHeaderSchema:"Schema",
+ MessageHeaderDictionaryBatch:"DictionaryBatch",
+ MessageHeaderRecordBatch:"RecordBatch",
+ MessageHeaderTensor:"Tensor",
+ MessageHeaderSparseTensor:"SparseTensor",
+}
+
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/MetadataVersion.go
similarity index 62%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/MetadataVersion.go
index 451f68e..0094430 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/MetadataVersion.go
@@ -14,11 +14,26 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type MetadataVersion = int16
+const (
+ /// 0.1.0
+ MetadataVersionV1 MetadataVersion = 0
+ /// 0.2.0
+ MetadataVersionV2 MetadataVersion = 1
+ /// 0.3.0 -> 0.7.1
+ MetadataVersionV3 MetadataVersion = 2
+ /// >= 0.8.0
+ MetadataVersionV4 MetadataVersion = 3
)
+
+var EnumNamesMetadataVersion = map[MetadataVersion]string{
+ MetadataVersionV1:"V1",
+ MetadataVersionV2:"V2",
+ MetadataVersionV3:"V3",
+ MetadataVersionV4:"V4",
+}
+
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/Null.go
similarity index 51%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/Null.go
index 451f68e..3c3eb4b 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/Null.go
@@ -14,11 +14,38 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
)
+
+/// These are stored in the flatbuffer in the Type union below
+type Null struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsNull(buf []byte, offset flatbuffers.UOffsetT) *Null {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Null{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Null) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Null) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func NullStart(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func NullEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/Precision.go
similarity index 70%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/Precision.go
index 451f68e..417d1e7 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/Precision.go
@@ -14,11 +14,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type Precision = int16
+const (
+ PrecisionHALF Precision = 0
+ PrecisionSINGLE Precision = 1
+ PrecisionDOUBLE Precision = 2
)
+
+var EnumNamesPrecision = map[Precision]string{
+ PrecisionHALF:"HALF",
+ PrecisionSINGLE:"SINGLE",
+ PrecisionDOUBLE:"DOUBLE",
+}
+
diff --git a/go/arrow/internal/flatbuf/RecordBatch.go b/go/arrow/internal/flatbuf/RecordBatch.go
new file mode 100644
index 0000000..b997bdf
--- /dev/null
+++ b/go/arrow/internal/flatbuf/RecordBatch.go
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// A data header describing the shared memory layout of a "record" or "row"
+/// batch. Some systems call this a "row batch" internally and others a "record
+/// batch".
+type RecordBatch struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsRecordBatch(buf []byte, offset flatbuffers.UOffsetT) *RecordBatch {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &RecordBatch{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *RecordBatch) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *RecordBatch) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// number of records / rows. The arrays in the batch should all have this
+/// length
+func (rcv *RecordBatch) Length() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// number of records / rows. The arrays in the batch should all have this
+/// length
+func (rcv *RecordBatch) MutateLength(n int64) bool {
+ return rcv._tab.MutateInt64Slot(4, n)
+}
+
+/// Nodes correspond to the pre-ordered flattened logical schema
+func (rcv *RecordBatch) Nodes(obj *FieldNode, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 16
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *RecordBatch) NodesLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+/// Nodes correspond to the pre-ordered flattened logical schema
+/// Buffers correspond to the pre-ordered flattened buffer tree
+///
+/// The number of buffers appended to this list depends on the schema. For
+/// example, most primitive arrays will have 2 buffers, 1 for the validity
+/// bitmap and 1 for the values. For struct arrays, there will only be a
+/// single buffer for the validity (nulls) bitmap
+func (rcv *RecordBatch) Buffers(obj *Buffer, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 16
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *RecordBatch) BuffersLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+/// Buffers correspond to the pre-ordered flattened buffer tree
+///
+/// The number of buffers appended to this list depends on the schema. For
+/// example, most primitive arrays will have 2 buffers, 1 for the validity
+/// bitmap and 1 for the values. For struct arrays, there will only be a
+/// single buffer for the validity (nulls) bitmap
+func RecordBatchStart(builder *flatbuffers.Builder) {
+ builder.StartObject(3)
+}
+func RecordBatchAddLength(builder *flatbuffers.Builder, length int64) {
+ builder.PrependInt64Slot(0, length, 0)
+}
+func RecordBatchAddNodes(builder *flatbuffers.Builder, nodes flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(nodes), 0)
+}
+func RecordBatchStartNodesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(16, numElems, 8)
+}
+func RecordBatchAddBuffers(builder *flatbuffers.Builder, buffers flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(buffers), 0)
+}
+func RecordBatchStartBuffersVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(16, numElems, 8)
+}
+func RecordBatchEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Schema.go b/go/arrow/internal/flatbuf/Schema.go
new file mode 100644
index 0000000..a8011c5
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Schema.go
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// A Schema describes the columns in a row batch
+type Schema struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsSchema(buf []byte, offset flatbuffers.UOffsetT) *Schema {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Schema{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Schema) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Schema) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// endianness of the buffer
+/// it is Little Endian by default
+/// if endianness doesn't match the underlying system then the vectors need to be converted
+func (rcv *Schema) Endianness() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// endianness of the buffer
+/// it is Little Endian by default
+/// if endianness doesn't match the underlying system then the vectors need to be converted
+func (rcv *Schema) MutateEndianness(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func (rcv *Schema) Fields(obj *Field, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Schema) FieldsLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func (rcv *Schema) CustomMetadata(obj *KeyValue, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Schema) CustomMetadataLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func SchemaStart(builder *flatbuffers.Builder) {
+ builder.StartObject(3)
+}
+func SchemaAddEndianness(builder *flatbuffers.Builder, endianness int16) {
+ builder.PrependInt16Slot(0, endianness, 0)
+}
+func SchemaAddFields(builder *flatbuffers.Builder, fields flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(fields), 0)
+}
+func SchemaStartFieldsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func SchemaAddCustomMetadata(builder *flatbuffers.Builder, customMetadata flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(customMetadata), 0)
+}
+func SchemaStartCustomMetadataVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func SchemaEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go b/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go
new file mode 100644
index 0000000..c09006c
--- /dev/null
+++ b/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// Compressed Sparse Row format, that is matrix-specific.
+type SparseMatrixIndexCSR struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsSparseMatrixIndexCSR(buf []byte, offset flatbuffers.UOffsetT) *SparseMatrixIndexCSR {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &SparseMatrixIndexCSR{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *SparseMatrixIndexCSR) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *SparseMatrixIndexCSR) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// indptrBuffer stores the location and size of indptr array that
+/// represents the range of the rows.
+/// The i-th row spans from indptr[i] to indptr[i+1] in the data.
+/// The length of this array is 1 + (the number of rows), and the type
+/// of index value is long.
+///
+/// For example, let X be the following 6x4 matrix:
+///
+/// X := [[0, 1, 2, 0],
+/// [0, 0, 3, 0],
+/// [0, 4, 0, 5],
+/// [0, 0, 0, 0],
+/// [6, 0, 7, 8],
+/// [0, 9, 0, 0]].
+///
+/// The array of non-zero values in X is:
+///
+/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9].
+///
+/// And the indptr of X is:
+///
+/// indptr(X) = [0, 2, 3, 5, 5, 8, 10].
+func (rcv *SparseMatrixIndexCSR) IndptrBuffer(obj *Buffer) *Buffer {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ x := o + rcv._tab.Pos
+ if obj == nil {
+ obj = new(Buffer)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// indptrBuffer stores the location and size of indptr array that
+/// represents the range of the rows.
+/// The i-th row spans from indptr[i] to indptr[i+1] in the data.
+/// The length of this array is 1 + (the number of rows), and the type
+/// of index value is long.
+///
+/// For example, let X be the following 6x4 matrix:
+///
+/// X := [[0, 1, 2, 0],
+/// [0, 0, 3, 0],
+/// [0, 4, 0, 5],
+/// [0, 0, 0, 0],
+/// [6, 0, 7, 8],
+/// [0, 9, 0, 0]].
+///
+/// The array of non-zero values in X is:
+///
+/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9].
+///
+/// And the indptr of X is:
+///
+/// indptr(X) = [0, 2, 3, 5, 5, 8, 10].
+/// indicesBuffer stores the location and size of the array that
+/// contains the column indices of the corresponding non-zero values.
+/// The type of index value is long.
+///
+/// For example, the indices of the above X is:
+///
+/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1].
+///
+/// Note that the indices are sorted in lexicographical order for each row.
+func (rcv *SparseMatrixIndexCSR) IndicesBuffer(obj *Buffer) *Buffer {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := o + rcv._tab.Pos
+ if obj == nil {
+ obj = new(Buffer)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// indicesBuffer stores the location and size of the array that
+/// contains the column indices of the corresponding non-zero values.
+/// The type of index value is long.
+///
+/// For example, the indices of the above X is:
+///
+/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1].
+///
+/// Note that the indices are sorted in lexicographical order for each row.
+func SparseMatrixIndexCSRStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func SparseMatrixIndexCSRAddIndptrBuffer(builder *flatbuffers.Builder, indptrBuffer flatbuffers.UOffsetT) {
+ builder.PrependStructSlot(0, flatbuffers.UOffsetT(indptrBuffer), 0)
+}
+func SparseMatrixIndexCSRAddIndicesBuffer(builder *flatbuffers.Builder, indicesBuffer flatbuffers.UOffsetT) {
+ builder.PrependStructSlot(1, flatbuffers.UOffsetT(indicesBuffer), 0)
+}
+func SparseMatrixIndexCSREnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/SparseTensor.go b/go/arrow/internal/flatbuf/SparseTensor.go
new file mode 100644
index 0000000..389832b
--- /dev/null
+++ b/go/arrow/internal/flatbuf/SparseTensor.go
@@ -0,0 +1,175 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type SparseTensor struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsSparseTensor(buf []byte, offset flatbuffers.UOffsetT) *SparseTensor {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &SparseTensor{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *SparseTensor) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *SparseTensor) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *SparseTensor) TypeType() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *SparseTensor) MutateTypeType(n byte) bool {
+ return rcv._tab.MutateByteSlot(4, n)
+}
+
+/// The type of data contained in a value cell.
+/// Currently only fixed-width value types are supported,
+/// no strings or nested types.
+func (rcv *SparseTensor) Type(obj *flatbuffers.Table) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ rcv._tab.Union(obj, o)
+ return true
+ }
+ return false
+}
+
+/// The type of data contained in a value cell.
+/// Currently only fixed-width value types are supported,
+/// no strings or nested types.
+/// The dimensions of the tensor, optionally named.
+func (rcv *SparseTensor) Shape(obj *TensorDim, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *SparseTensor) ShapeLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+/// The dimensions of the tensor, optionally named.
+/// The number of non-zero values in a sparse tensor.
+func (rcv *SparseTensor) NonZeroLength() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// The number of non-zero values in a sparse tensor.
+func (rcv *SparseTensor) MutateNonZeroLength(n int64) bool {
+ return rcv._tab.MutateInt64Slot(10, n)
+}
+
+func (rcv *SparseTensor) SparseIndexType() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *SparseTensor) MutateSparseIndexType(n byte) bool {
+ return rcv._tab.MutateByteSlot(12, n)
+}
+
+/// Sparse tensor index
+func (rcv *SparseTensor) SparseIndex(obj *flatbuffers.Table) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
+ if o != 0 {
+ rcv._tab.Union(obj, o)
+ return true
+ }
+ return false
+}
+
+/// Sparse tensor index
+/// The location and size of the tensor's data
+func (rcv *SparseTensor) Data(obj *Buffer) *Buffer {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(16))
+ if o != 0 {
+ x := o + rcv._tab.Pos
+ if obj == nil {
+ obj = new(Buffer)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// The location and size of the tensor's data
+func SparseTensorStart(builder *flatbuffers.Builder) {
+ builder.StartObject(7)
+}
+func SparseTensorAddTypeType(builder *flatbuffers.Builder, typeType byte) {
+ builder.PrependByteSlot(0, typeType, 0)
+}
+func SparseTensorAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(type_), 0)
+}
+func SparseTensorAddShape(builder *flatbuffers.Builder, shape flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(shape), 0)
+}
+func SparseTensorStartShapeVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func SparseTensorAddNonZeroLength(builder *flatbuffers.Builder, nonZeroLength int64) {
+ builder.PrependInt64Slot(3, nonZeroLength, 0)
+}
+func SparseTensorAddSparseIndexType(builder *flatbuffers.Builder, sparseIndexType byte) {
+ builder.PrependByteSlot(4, sparseIndexType, 0)
+}
+func SparseTensorAddSparseIndex(builder *flatbuffers.Builder, sparseIndex flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(sparseIndex), 0)
+}
+func SparseTensorAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) {
+ builder.PrependStructSlot(6, flatbuffers.UOffsetT(data), 0)
+}
+func SparseTensorEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/SparseTensorIndex.go
similarity index 61%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/SparseTensorIndex.go
index 451f68e..4b82de7 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/SparseTensorIndex.go
@@ -14,11 +14,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type SparseTensorIndex = byte
+const (
+ SparseTensorIndexNONE SparseTensorIndex = 0
+ SparseTensorIndexSparseTensorIndexCOO SparseTensorIndex = 1
+ SparseTensorIndexSparseMatrixIndexCSR SparseTensorIndex = 2
)
+
+var EnumNamesSparseTensorIndex = map[SparseTensorIndex]string{
+ SparseTensorIndexNONE:"NONE",
+ SparseTensorIndexSparseTensorIndexCOO:"SparseTensorIndexCOO",
+ SparseTensorIndexSparseMatrixIndexCSR:"SparseMatrixIndexCSR",
+}
+
diff --git a/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go b/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go
new file mode 100644
index 0000000..6d8a0e8
--- /dev/null
+++ b/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// EXPERIMENTAL: Data structures for sparse tensors
+/// Coodinate format of sparse tensor index.
+type SparseTensorIndexCOO struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsSparseTensorIndexCOO(buf []byte, offset flatbuffers.UOffsetT) *SparseTensorIndexCOO {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &SparseTensorIndexCOO{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *SparseTensorIndexCOO) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *SparseTensorIndexCOO) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// COO's index list are represented as a NxM matrix,
+/// where N is the number of non-zero values,
+/// and M is the number of dimensions of a sparse tensor.
+/// indicesBuffer stores the location and size of this index matrix.
+/// The type of index value is long, so the stride for the index matrix is unnecessary.
+///
+/// For example, let X be a 2x3x4x5 tensor, and it has the following 6 non-zero values:
+///
+/// X[0, 1, 2, 0] := 1
+/// X[1, 1, 2, 3] := 2
+/// X[0, 2, 1, 0] := 3
+/// X[0, 1, 3, 0] := 4
+/// X[0, 1, 2, 1] := 5
+/// X[1, 2, 0, 4] := 6
+///
+/// In COO format, the index matrix of X is the following 4x6 matrix:
+///
+/// [[0, 0, 0, 0, 1, 1],
+/// [1, 1, 1, 2, 1, 2],
+/// [2, 2, 3, 1, 2, 0],
+/// [0, 1, 0, 0, 3, 4]]
+///
+/// Note that the indices are sorted in lexicographical order.
+func (rcv *SparseTensorIndexCOO) IndicesBuffer(obj *Buffer) *Buffer {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ x := o + rcv._tab.Pos
+ if obj == nil {
+ obj = new(Buffer)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// COO's index list are represented as a NxM matrix,
+/// where N is the number of non-zero values,
+/// and M is the number of dimensions of a sparse tensor.
+/// indicesBuffer stores the location and size of this index matrix.
+/// The type of index value is long, so the stride for the index matrix is unnecessary.
+///
+/// For example, let X be a 2x3x4x5 tensor, and it has the following 6 non-zero values:
+///
+/// X[0, 1, 2, 0] := 1
+/// X[1, 1, 2, 3] := 2
+/// X[0, 2, 1, 0] := 3
+/// X[0, 1, 3, 0] := 4
+/// X[0, 1, 2, 1] := 5
+/// X[1, 2, 0, 4] := 6
+///
+/// In COO format, the index matrix of X is the following 4x6 matrix:
+///
+/// [[0, 0, 0, 0, 1, 1],
+/// [1, 1, 1, 2, 1, 2],
+/// [2, 2, 3, 1, 2, 0],
+/// [0, 1, 0, 0, 3, 4]]
+///
+/// Note that the indices are sorted in lexicographical order.
+func SparseTensorIndexCOOStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func SparseTensorIndexCOOAddIndicesBuffer(builder *flatbuffers.Builder, indicesBuffer flatbuffers.UOffsetT) {
+ builder.PrependStructSlot(0, flatbuffers.UOffsetT(indicesBuffer), 0)
+}
+func SparseTensorIndexCOOEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Struct_.go b/go/arrow/internal/flatbuf/Struct_.go
new file mode 100644
index 0000000..427e706
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Struct_.go
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
+/// (according to the physical memory layout). We used Struct_ here as
+/// Struct is a reserved word in Flatbuffers
+type Struct_ struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsStruct_(buf []byte, offset flatbuffers.UOffsetT) *Struct_ {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Struct_{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Struct_) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Struct_) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func Struct_Start(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func Struct_End(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Tensor.go b/go/arrow/internal/flatbuf/Tensor.go
new file mode 100644
index 0000000..73e50b5
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Tensor.go
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Tensor struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsTensor(buf []byte, offset flatbuffers.UOffsetT) *Tensor {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Tensor{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Tensor) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Tensor) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Tensor) TypeType() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Tensor) MutateTypeType(n byte) bool {
+ return rcv._tab.MutateByteSlot(4, n)
+}
+
+/// The type of data contained in a value cell. Currently only fixed-width
+/// value types are supported, no strings or nested types
+func (rcv *Tensor) Type(obj *flatbuffers.Table) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ rcv._tab.Union(obj, o)
+ return true
+ }
+ return false
+}
+
+/// The type of data contained in a value cell. Currently only fixed-width
+/// value types are supported, no strings or nested types
+/// The dimensions of the tensor, optionally named
+func (rcv *Tensor) Shape(obj *TensorDim, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *Tensor) ShapeLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+/// The dimensions of the tensor, optionally named
+/// Non-negative byte offsets to advance one value cell along each dimension
+func (rcv *Tensor) Strides(j int) int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ a := rcv._tab.Vector(o)
+ return rcv._tab.GetInt64(a + flatbuffers.UOffsetT(j*8))
+ }
+ return 0
+}
+
+func (rcv *Tensor) StridesLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+/// Non-negative byte offsets to advance one value cell along each dimension
+/// The location and size of the tensor's data
+func (rcv *Tensor) Data(obj *Buffer) *Buffer {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ x := o + rcv._tab.Pos
+ if obj == nil {
+ obj = new(Buffer)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// The location and size of the tensor's data
+func TensorStart(builder *flatbuffers.Builder) {
+ builder.StartObject(5)
+}
+func TensorAddTypeType(builder *flatbuffers.Builder, typeType byte) {
+ builder.PrependByteSlot(0, typeType, 0)
+}
+func TensorAddType(builder *flatbuffers.Builder, type_ flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(type_), 0)
+}
+func TensorAddShape(builder *flatbuffers.Builder, shape flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(shape), 0)
+}
+func TensorStartShapeVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func TensorAddStrides(builder *flatbuffers.Builder, strides flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(strides), 0)
+}
+func TensorStartStridesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(8, numElems, 8)
+}
+func TensorAddData(builder *flatbuffers.Builder, data flatbuffers.UOffsetT) {
+ builder.PrependStructSlot(4, flatbuffers.UOffsetT(data), 0)
+}
+func TensorEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/TensorDim.go b/go/arrow/internal/flatbuf/TensorDim.go
new file mode 100644
index 0000000..14b8212
--- /dev/null
+++ b/go/arrow/internal/flatbuf/TensorDim.go
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// ----------------------------------------------------------------------
+/// Data structures for dense tensors
+/// Shape data for a single axis in a tensor
+type TensorDim struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsTensorDim(buf []byte, offset flatbuffers.UOffsetT) *TensorDim {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &TensorDim{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *TensorDim) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *TensorDim) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// Length of dimension
+func (rcv *TensorDim) Size() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Length of dimension
+func (rcv *TensorDim) MutateSize(n int64) bool {
+ return rcv._tab.MutateInt64Slot(4, n)
+}
+
+/// Name of the dimension, optional
+func (rcv *TensorDim) Name() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+/// Name of the dimension, optional
+func TensorDimStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func TensorDimAddSize(builder *flatbuffers.Builder, size int64) {
+ builder.PrependInt64Slot(0, size, 0)
+}
+func TensorDimAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(name), 0)
+}
+func TensorDimEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Time.go b/go/arrow/internal/flatbuf/Time.go
new file mode 100644
index 0000000..3f3b7fc
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Time.go
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// Time type. The physical storage type depends on the unit
+/// - SECOND and MILLISECOND: 32 bits
+/// - MICROSECOND and NANOSECOND: 64 bits
+type Time struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsTime(buf []byte, offset flatbuffers.UOffsetT) *Time {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Time{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Time) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Time) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Time) Unit() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 1
+}
+
+func (rcv *Time) MutateUnit(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func (rcv *Time) BitWidth() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 32
+}
+
+func (rcv *Time) MutateBitWidth(n int32) bool {
+ return rcv._tab.MutateInt32Slot(6, n)
+}
+
+func TimeStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func TimeAddUnit(builder *flatbuffers.Builder, unit int16) {
+ builder.PrependInt16Slot(0, unit, 1)
+}
+func TimeAddBitWidth(builder *flatbuffers.Builder, bitWidth int32) {
+ builder.PrependInt32Slot(1, bitWidth, 32)
+}
+func TimeEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/TimeUnit.go
similarity index 65%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/TimeUnit.go
index 451f68e..cb0c742 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/TimeUnit.go
@@ -14,11 +14,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type TimeUnit = int16
+const (
+ TimeUnitSECOND TimeUnit = 0
+ TimeUnitMILLISECOND TimeUnit = 1
+ TimeUnitMICROSECOND TimeUnit = 2
+ TimeUnitNANOSECOND TimeUnit = 3
)
+
+var EnumNamesTimeUnit = map[TimeUnit]string{
+ TimeUnitSECOND:"SECOND",
+ TimeUnitMILLISECOND:"MILLISECOND",
+ TimeUnitMICROSECOND:"MICROSECOND",
+ TimeUnitNANOSECOND:"NANOSECOND",
+}
+
diff --git a/go/arrow/internal/flatbuf/Timestamp.go b/go/arrow/internal/flatbuf/Timestamp.go
new file mode 100644
index 0000000..ba96629
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Timestamp.go
@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding
+/// leap seconds, as a 64-bit integer. Note that UNIX time does not include
+/// leap seconds.
+///
+/// The Timestamp metadata supports both "time zone naive" and "time zone
+/// aware" timestamps. Read about the timezone attribute for more detail
+type Timestamp struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsTimestamp(buf []byte, offset flatbuffers.UOffsetT) *Timestamp {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Timestamp{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Timestamp) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Timestamp) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Timestamp) Unit() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Timestamp) MutateUnit(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+/// The time zone is a string indicating the name of a time zone, one of:
+///
+/// * As used in the Olson time zone database (the "tz database" or
+/// "tzdata"), such as "America/New_York"
+/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+///
+/// Whether a timezone string is present indicates different semantics about
+/// the data:
+///
+/// * If the time zone is null or equal to an empty string, the data is "time
+/// zone naive" and shall be displayed *as is* to the user, not localized
+/// to the locale of the user. This data can be though of as UTC but
+/// without having "UTC" as the time zone, it is not considered to be
+/// localized to any time zone
+///
+/// * If the time zone is set to a valid value, values can be displayed as
+/// "localized" to that time zone, even though the underlying 64-bit
+/// integers are identical to the same data stored in UTC. Converting
+/// between time zones is a metadata-only operation and does not change the
+/// underlying values
+func (rcv *Timestamp) Timezone() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+/// The time zone is a string indicating the name of a time zone, one of:
+///
+/// * As used in the Olson time zone database (the "tz database" or
+/// "tzdata"), such as "America/New_York"
+/// * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30
+///
+/// Whether a timezone string is present indicates different semantics about
+/// the data:
+///
+/// * If the time zone is null or equal to an empty string, the data is "time
+/// zone naive" and shall be displayed *as is* to the user, not localized
+/// to the locale of the user. This data can be though of as UTC but
+/// without having "UTC" as the time zone, it is not considered to be
+/// localized to any time zone
+///
+/// * If the time zone is set to a valid value, values can be displayed as
+/// "localized" to that time zone, even though the underlying 64-bit
+/// integers are identical to the same data stored in UTC. Converting
+/// between time zones is a metadata-only operation and does not change the
+/// underlying values
+func TimestampStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func TimestampAddUnit(builder *flatbuffers.Builder, unit int16) {
+ builder.PrependInt16Slot(0, unit, 0)
+}
+func TimestampAddTimezone(builder *flatbuffers.Builder, timezone flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(timezone), 0)
+}
+func TimestampEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/internal/flatbuf/Type.go b/go/arrow/internal/flatbuf/Type.go
new file mode 100644
index 0000000..c8bb7f8
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Type.go
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+/// ----------------------------------------------------------------------
+/// Top-level Type value, enabling extensible type-specific metadata. We can
+/// add new logical types to Type without breaking backwards compatibility
+type Type = byte
+const (
+ TypeNONE Type = 0
+ TypeNull Type = 1
+ TypeInt Type = 2
+ TypeFloatingPoint Type = 3
+ TypeBinary Type = 4
+ TypeUtf8 Type = 5
+ TypeBool Type = 6
+ TypeDecimal Type = 7
+ TypeDate Type = 8
+ TypeTime Type = 9
+ TypeTimestamp Type = 10
+ TypeInterval Type = 11
+ TypeList Type = 12
+ TypeStruct_ Type = 13
+ TypeUnion Type = 14
+ TypeFixedSizeBinary Type = 15
+ TypeFixedSizeList Type = 16
+ TypeMap Type = 17
+)
+
+var EnumNamesType = map[Type]string{
+ TypeNONE:"NONE",
+ TypeNull:"Null",
+ TypeInt:"Int",
+ TypeFloatingPoint:"FloatingPoint",
+ TypeBinary:"Binary",
+ TypeUtf8:"Utf8",
+ TypeBool:"Bool",
+ TypeDecimal:"Decimal",
+ TypeDate:"Date",
+ TypeTime:"Time",
+ TypeTimestamp:"Timestamp",
+ TypeInterval:"Interval",
+ TypeList:"List",
+ TypeStruct_:"Struct_",
+ TypeUnion:"Union",
+ TypeFixedSizeBinary:"FixedSizeBinary",
+ TypeFixedSizeList:"FixedSizeList",
+ TypeMap:"Map",
+}
+
diff --git a/go/arrow/internal/flatbuf/Union.go b/go/arrow/internal/flatbuf/Union.go
new file mode 100644
index 0000000..ccf988a
--- /dev/null
+++ b/go/arrow/internal/flatbuf/Union.go
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+/// A union is a complex type with children in Field
+/// By default ids in the type vector refer to the offsets in the children
+/// optionally typeIds provides an indirection between the child offset and the type id
+/// for each child typeIds[offset] is the id used in the type vector
+type Union struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsUnion(buf []byte, offset flatbuffers.UOffsetT) *Union {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Union{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Union) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Union) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Union) Mode() int16 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt16(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Union) MutateMode(n int16) bool {
+ return rcv._tab.MutateInt16Slot(4, n)
+}
+
+func (rcv *Union) TypeIds(j int) int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ a := rcv._tab.Vector(o)
+ return rcv._tab.GetInt32(a + flatbuffers.UOffsetT(j*4))
+ }
+ return 0
+}
+
+func (rcv *Union) TypeIdsLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+func UnionStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func UnionAddMode(builder *flatbuffers.Builder, mode int16) {
+ builder.PrependInt16Slot(0, mode, 0)
+}
+func UnionAddTypeIds(builder *flatbuffers.Builder, typeIds flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(typeIds), 0)
+}
+func UnionStartTypeIdsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func UnionEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/UnionMode.go
similarity index 74%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/UnionMode.go
index 451f68e..4357682 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/UnionMode.go
@@ -14,11 +14,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+type UnionMode = int16
+const (
+ UnionModeSparse UnionMode = 0
+ UnionModeDense UnionMode = 1
)
+
+var EnumNamesUnionMode = map[UnionMode]string{
+ UnionModeSparse:"Sparse",
+ UnionModeDense:"Dense",
+}
+
diff --git a/go/arrow/go.mod b/go/arrow/internal/flatbuf/Utf8.go
similarity index 52%
copy from go/arrow/go.mod
copy to go/arrow/internal/flatbuf/Utf8.go
index 451f68e..4ff365a 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/internal/flatbuf/Utf8.go
@@ -14,11 +14,38 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package flatbuf
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
)
+
+/// Unicode with UTF-8 encoding
+type Utf8 struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsUtf8(buf []byte, offset flatbuffers.UOffsetT) *Utf8 {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Utf8{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Utf8) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Utf8) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func Utf8Start(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func Utf8End(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go
new file mode 100644
index 0000000..cb4ff5a
--- /dev/null
+++ b/go/arrow/ipc/cmd/arrow-cat/main.go
@@ -0,0 +1,210 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command arrow-cat displays the content of an Arrow stream or file.
+//
+// Examples:
+//
+// $> arrow-cat ./testdata/primitives.data
+// version: V4
+// record 1/3...
+// col[0] "bools": [true (null) (null) false true]
+// col[1] "int8s": [-1 (null) (null) -4 -5]
+// col[2] "int16s": [-1 (null) (null) -4 -5]
+// col[3] "int32s": [-1 (null) (null) -4 -5]
+// col[4] "int64s": [-1 (null) (null) -4 -5]
+// col[5] "uint8s": [1 (null) (null) 4 5]
+// col[6] "uint16s": [1 (null) (null) 4 5]
+// col[7] "uint32s": [1 (null) (null) 4 5]
+// col[8] "uint64s": [1 (null) (null) 4 5]
+// col[9] "float32s": [1 (null) (null) 4 5]
+// col[10] "float64s": [1 (null) (null) 4 5]
+// record 2/3...
+// col[0] "bools": [true (null) (null) false true]
+// [...]
+//
+// $> gen-arrow-stream | arrow-cat
+// record 1...
+// col[0] "bools": [true (null) (null) false true]
+// col[1] "int8s": [-1 (null) (null) -4 -5]
+// col[2] "int16s": [-1 (null) (null) -4 -5]
+// col[3] "int32s": [-1 (null) (null) -4 -5]
+// col[4] "int64s": [-1 (null) (null) -4 -5]
+// col[5] "uint8s": [1 (null) (null) 4 5]
+// col[6] "uint16s": [1 (null) (null) 4 5]
+// col[7] "uint32s": [1 (null) (null) 4 5]
+// col[8] "uint64s": [1 (null) (null) 4 5]
+// col[9] "float32s": [1 (null) (null) 4 5]
+// col[10] "float64s": [1 (null) (null) 4 5]
+// record 2...
+// col[0] "bools": [true (null) (null) false true]
+// [...]
+package main // import "github.com/apache/arrow/go/arrow/ipc/cmd/arrow-cat"
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+
+ "github.com/apache/arrow/go/arrow/ipc"
+ "github.com/apache/arrow/go/arrow/memory"
+ "github.com/pkg/errors"
+)
+
+func main() {
+ log.SetPrefix("arrow-cat: ")
+ log.SetFlags(0)
+
+ flag.Parse()
+
+ var err error
+ switch flag.NArg() {
+ case 0:
+ err = processStream(os.Stdout, os.Stdin)
+ default:
+ err = processFiles(os.Stdout, flag.Args())
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func processStream(w io.Writer, rin io.Reader) error {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(nil, 0)
+
+ r, err := ipc.NewReader(rin, ipc.WithAllocator(mem))
+ if err != nil {
+ return err
+ }
+ defer r.Release()
+
+ n := 0
+ for r.Next() {
+ n++
+ fmt.Fprintf(w, "record %d...\n", n)
+ rec := r.Record()
+ for i, col := range rec.Columns() {
+ fmt.Fprintf(w, " col[%d] %q: %v\n", i, rec.ColumnName(i), col)
+ }
+ }
+ return nil
+}
+
+func processFiles(w io.Writer, names []string) error {
+ for _, name := range names {
+ err := processFile(w, name)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func processFile(w io.Writer, fname string) error {
+
+ f, err := os.Open(fname)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ hdr := make([]byte, len(ipc.Magic))
+ _, err = io.ReadFull(f, hdr)
+ if err != nil {
+ return errors.Errorf("could not read file header: %v", err)
+ }
+ f.Seek(0, io.SeekStart)
+
+ if !bytes.Equal(hdr, ipc.Magic) {
+ // try as a stream.
+ return processStream(w, f)
+ }
+
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(nil, 0)
+
+ r, err := ipc.NewFileReader(f, ipc.WithAllocator(mem))
+ if err != nil {
+ return err
+ }
+ defer r.Close()
+
+ fmt.Fprintf(w, "version: %v\n", r.Version())
+ for i := 0; i < r.NumRecords(); i++ {
+ fmt.Fprintf(w, "record %d/%d...\n", i+1, r.NumRecords())
+ rec, err := r.Record(i)
+ if err != nil {
+ return err
+ }
+ defer rec.Release()
+
+ for i, col := range rec.Columns() {
+ fmt.Fprintf(w, " col[%d] %q: %v\n", i, rec.ColumnName(i), col)
+ }
+ }
+ return nil
+}
+
+func init() {
+ flag.Usage = func() {
+ fmt.Fprintf(os.Stderr, `Command arrow-cat displays the content of an Arrow stream or file.
+
+Usage: arrow-cat [OPTIONS] [FILE1 [FILE2 [...]]]
+
+Examples:
+
+ $> arrow-cat ./testdata/primitives.data
+ version: V4
+ record 1/3...
+ col[0] "bools": [true (null) (null) false true]
+ col[1] "int8s": [-1 (null) (null) -4 -5]
+ col[2] "int16s": [-1 (null) (null) -4 -5]
+ col[3] "int32s": [-1 (null) (null) -4 -5]
+ col[4] "int64s": [-1 (null) (null) -4 -5]
+ col[5] "uint8s": [1 (null) (null) 4 5]
+ col[6] "uint16s": [1 (null) (null) 4 5]
+ col[7] "uint32s": [1 (null) (null) 4 5]
+ col[8] "uint64s": [1 (null) (null) 4 5]
+ col[9] "float32s": [1 (null) (null) 4 5]
+ col[10] "float64s": [1 (null) (null) 4 5]
+ record 2/3...
+ col[0] "bools": [true (null) (null) false true]
+ [...]
+
+ $> gen-arrow-stream | arrow-cat
+ record 1...
+ col[0] "bools": [true (null) (null) false true]
+ col[1] "int8s": [-1 (null) (null) -4 -5]
+ col[2] "int16s": [-1 (null) (null) -4 -5]
+ col[3] "int32s": [-1 (null) (null) -4 -5]
+ col[4] "int64s": [-1 (null) (null) -4 -5]
+ col[5] "uint8s": [1 (null) (null) 4 5]
+ col[6] "uint16s": [1 (null) (null) 4 5]
+ col[7] "uint32s": [1 (null) (null) 4 5]
+ col[8] "uint64s": [1 (null) (null) 4 5]
+ col[9] "float32s": [1 (null) (null) 4 5]
+ col[10] "float64s": [1 (null) (null) 4 5]
+ record 2...
+ col[0] "bools": [true (null) (null) false true]
+ [...]
+`)
+ os.Exit(0)
+ }
+}
diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go
new file mode 100644
index 0000000..b858bf5
--- /dev/null
+++ b/go/arrow/ipc/cmd/arrow-ls/main.go
@@ -0,0 +1,205 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command arrow-ls displays the listing of an Arrow file.
+//
+// Examples:
+//
+// $> arrow-ls ./testdata/primitives.data
+// version: V4
+// schema:
+// fields: 11
+// - bools: type=bool, nullable
+// - int8s: type=int8, nullable
+// - int16s: type=int16, nullable
+// - int32s: type=int32, nullable
+// - int64s: type=int64, nullable
+// - uint8s: type=uint8, nullable
+// - uint16s: type=uint16, nullable
+// - uint32s: type=uint32, nullable
+// - uint64s: type=uint64, nullable
+// - float32s: type=float32, nullable
+// - float64s: type=float64, nullable
+// records: 3
+//
+// $> gen-arrow-stream | arrow-ls
+// schema:
+// fields: 11
+// - bools: type=bool, nullable
+// - int8s: type=int8, nullable
+// - int16s: type=int16, nullable
+// - int32s: type=int32, nullable
+// - int64s: type=int64, nullable
+// - uint8s: type=uint8, nullable
+// - uint16s: type=uint16, nullable
+// - uint32s: type=uint32, nullable
+// - uint64s: type=uint64, nullable
+// - float32s: type=float32, nullable
+// - float64s: type=float64, nullable
+// records: 3
+package main // import "github.com/apache/arrow/go/arrow/ipc/cmd/arrow-ls"
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "strings"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/ipc"
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+func main() {
+ log.SetPrefix("arrow-ls: ")
+ log.SetFlags(0)
+
+ flag.Parse()
+
+ var err error
+ switch flag.NArg() {
+ case 0:
+ err = processStream(os.Stdout, os.Stdin)
+ default:
+ err = processFiles(os.Stdout, flag.Args())
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+}
+
+func processStream(w io.Writer, rin io.Reader) error {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(nil, 0)
+
+ r, err := ipc.NewReader(rin, ipc.WithAllocator(mem))
+ if err != nil {
+ return err
+ }
+ defer r.Release()
+
+ fmt.Fprintf(w, "schema:\n%v", displaySchema(r.Schema()))
+
+ nrecs := 0
+ for r.Next() {
+ nrecs++
+ }
+ fmt.Fprintf(w, "records: %d\n", nrecs)
+ return nil
+}
+
+func processFiles(w io.Writer, names []string) error {
+ for _, name := range names {
+ err := processFile(w, name)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func processFile(w io.Writer, fname string) error {
+
+ f, err := os.Open(fname)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(nil, 0)
+
+ r, err := ipc.NewFileReader(f, ipc.WithAllocator(mem))
+ if err != nil {
+ return err
+ }
+ defer r.Close()
+
+ fmt.Fprintf(w, "version: %v\n", r.Version())
+ fmt.Fprintf(w, "schema:\n%v", displaySchema(r.Schema()))
+ fmt.Fprintf(w, "records: %d\n", r.NumRecords())
+ return nil
+}
+
+func displaySchema(s *arrow.Schema) string {
+ o := new(strings.Builder)
+ fmt.Fprintf(o, "%*.sfields: %d\n", 2, "", len(s.Fields()))
+ for _, f := range s.Fields() {
+ displayField(o, f, 4)
+ }
+ if meta := s.Metadata(); meta.Len() > 0 {
+ fmt.Fprintf(o, "metadata: %v\n", meta)
+ }
+ return o.String()
+}
+
+func displayField(o io.Writer, field arrow.Field, inc int) {
+ nullable := ""
+ if field.Nullable {
+ nullable = ", nullable"
+ }
+ fmt.Fprintf(o, "%*.s- %s: type=%v%v\n", inc, "", field.Name, field.Type.Name(), nullable)
+ if field.HasMetadata() {
+ fmt.Fprintf(o, "%*.smetadata: %v\n", inc, "", field.Metadata)
+ }
+}
+
+func init() {
+ flag.Usage = func() {
+ fmt.Fprintf(os.Stderr, `Command arrow-ls displays the listing of an Arrow file.
+
+Usage: arrow-ls [OPTIONS] [FILE1 [FILE2 [...]]]
+
+Examples:
+
+ $> arrow-ls ./testdata/primitives.data
+ version: V4
+ schema:
+ fields: 11
+ - bools: type=bool, nullable
+ - int8s: type=int8, nullable
+ - int16s: type=int16, nullable
+ - int32s: type=int32, nullable
+ - int64s: type=int64, nullable
+ - uint8s: type=uint8, nullable
+ - uint16s: type=uint16, nullable
+ - uint32s: type=uint32, nullable
+ - uint64s: type=uint64, nullable
+ - float32s: type=float32, nullable
+ - float64s: type=float64, nullable
+ records: 3
+
+ $> gen-arrow-stream | arrow-ls
+ schema:
+ fields: 11
+ - bools: type=bool, nullable
+ - int8s: type=int8, nullable
+ - int16s: type=int16, nullable
+ - int32s: type=int32, nullable
+ - int64s: type=int64, nullable
+ - uint8s: type=uint8, nullable
+ - uint16s: type=uint16, nullable
+ - uint32s: type=uint32, nullable
+ - uint64s: type=uint64, nullable
+ - float32s: type=float32, nullable
+ - float64s: type=float64, nullable
+ records: 3
+`)
+ os.Exit(0)
+ }
+}
diff --git a/go/arrow/ipc/dict.go b/go/arrow/ipc/dict.go
new file mode 100644
index 0000000..802a5ac
--- /dev/null
+++ b/go/arrow/ipc/dict.go
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/array"
+ "github.com/pkg/errors"
+)
+
+type dictMap map[int64]array.Interface
+type dictTypeMap map[int64]arrow.Field
+
+type dictMemo struct {
+ dict2id map[array.Interface]int64
+ id2dict dictMap // map of dictionary ID to dictionary array
+}
+
+func newMemo() dictMemo {
+ return dictMemo{
+ dict2id: make(map[array.Interface]int64),
+ id2dict: make(dictMap),
+ }
+}
+
+func (memo *dictMemo) Len() int { return len(memo.id2dict) }
+
+func (memo *dictMemo) delete() {
+ for id, v := range memo.id2dict {
+ delete(memo.id2dict, id)
+ delete(memo.dict2id, v)
+ v.Release()
+ }
+}
+
+func (memo dictMemo) Dict(id int64) (array.Interface, bool) {
+ v, ok := memo.id2dict[id]
+ return v, ok
+}
+
+func (memo *dictMemo) ID(v array.Interface) int64 {
+ id, ok := memo.dict2id[v]
+ if ok {
+ return id
+ }
+
+ v.Retain()
+ id = int64(len(memo.dict2id))
+ memo.dict2id[v] = id
+ memo.id2dict[id] = v
+ return id
+}
+
+func (memo dictMemo) HasDict(v array.Interface) bool {
+ _, ok := memo.dict2id[v]
+ return ok
+}
+
+func (memo dictMemo) HasID(id int64) bool {
+ _, ok := memo.id2dict[id]
+ return ok
+}
+
+func (memo *dictMemo) Add(id int64, v array.Interface) {
+ if _, dup := memo.id2dict[id]; dup {
+ panic(errors.Errorf("arrow/ipc: duplicate id=%d", id))
+ }
+ v.Retain()
+ memo.id2dict[id] = v
+ memo.dict2id[v] = id
+}
diff --git a/go/arrow/ipc/dict_test.go b/go/arrow/ipc/dict_test.go
new file mode 100644
index 0000000..ba51184
--- /dev/null
+++ b/go/arrow/ipc/dict_test.go
@@ -0,0 +1,196 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/arrow/go/arrow/array"
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+func TestDictMemo(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewFloat64Builder(mem)
+ defer bldr.Release()
+
+ bldr.AppendValues([]float64{1.0, 1.1, 1.2, 1.3}, nil)
+ f0 := bldr.NewFloat64Array()
+ defer f0.Release()
+
+ bldr.AppendValues([]float64{11.0, 11.1, 11.2, 11.3}, nil)
+ f1 := bldr.NewFloat64Array()
+ defer f1.Release()
+
+ bldr.AppendValues([]float64{11.0, 11.1, 11.2, 11.3}, nil)
+ f2 := bldr.NewFloat64Array()
+ defer f2.Release()
+
+ memo := newMemo()
+ defer memo.delete()
+
+ if got, want := memo.Len(), 0; got != want {
+ t.Fatalf("invalid length: got=%d, want=%d", got, want)
+ }
+
+ memo.Add(0, f0)
+ memo.Add(1, f1)
+
+ if !memo.HasID(0) {
+ t.Fatalf("could not find id=0")
+ }
+
+ if !memo.HasID(1) {
+ t.Fatalf("could not find id=1")
+ }
+
+ if got, want := memo.Len(), 2; got != want {
+ t.Fatalf("invalid length: got=%d, want=%d", got, want)
+ }
+
+ var ff array.Interface
+
+ ff = f0
+ if !memo.HasDict(ff) {
+ t.Fatalf("failed to find f0 through interface")
+ }
+
+ ff = f1
+ if !memo.HasDict(ff) {
+ t.Fatalf("failed to find f1 through interface")
+ }
+
+ ff = f2
+ if memo.HasDict(ff) {
+ t.Fatalf("should not have found f2")
+ }
+
+ fct := func(v array.Interface) array.Interface {
+ return v
+ }
+
+ if !memo.HasDict(fct(f1)) {
+ t.Fatalf("failed to find dict through func through interface")
+ }
+
+ if memo.HasDict(f2) {
+ t.Fatalf("should not have found f2")
+ }
+
+ ff = f0
+ for i, f := range []array.Interface{f0, f1, ff, fct(f0), fct(f1)} {
+ if !memo.HasDict(f) {
+ t.Fatalf("failed to find dict %d", i)
+ }
+ }
+
+ v, ok := memo.Dict(0)
+ if !ok {
+ t.Fatalf("expected to find id=0")
+ }
+ if v != f0 {
+ t.Fatalf("expected fo find id=0 array")
+ }
+
+ v, ok = memo.Dict(2)
+ if ok {
+ t.Fatalf("should not have found id=2")
+ }
+ v, ok = memo.Dict(-2)
+ if ok {
+ t.Fatalf("should not have found id=-2")
+ }
+
+ if got, want := memo.ID(f0), int64(0); got != want {
+ t.Fatalf("found invalid id. got=%d, want=%d", got, want)
+ }
+
+ if got, want := memo.ID(f2), int64(2); got != want {
+ t.Fatalf("found invalid id. got=%d, want=%d", got, want)
+ }
+ if !memo.HasDict(f2) {
+ t.Fatalf("should have found f2")
+ }
+
+ // test we don't leak nor "double-delete" when adding an array multiple times.
+ memo.Add(42, f2)
+ if got, want := memo.ID(f2), int64(42); got != want {
+ t.Fatalf("found invalid id. got=%d, want=%d", got, want)
+ }
+ memo.Add(43, f2)
+ if got, want := memo.ID(f2), int64(43); got != want {
+ t.Fatalf("found invalid id. got=%d, want=%d", got, want)
+ }
+ if got, want := memo.Len(), 5; got != want {
+ t.Fatalf("invalid length. got=%d, want=%d", got, want)
+ }
+}
+
+func TestDictMemoPanics(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ bldr := array.NewFloat64Builder(mem)
+ defer bldr.Release()
+
+ bldr.AppendValues([]float64{1.0, 1.1, 1.2, 1.3}, nil)
+ f0 := bldr.NewFloat64Array()
+ defer f0.Release()
+
+ bldr.AppendValues([]float64{11.0, 11.1, 11.2, 11.3}, nil)
+ f1 := bldr.NewFloat64Array()
+ defer f1.Release()
+
+ for _, tc := range []struct {
+ vs []array.Interface
+ ids []int64
+ }{
+ {
+ vs: []array.Interface{f0, f1},
+ ids: []int64{0, 0},
+ },
+ {
+ vs: []array.Interface{f0, f0},
+ ids: []int64{0, 0},
+ },
+ } {
+ t.Run("", func(t *testing.T) {
+ defer func() {
+ e := recover()
+ if e == nil {
+ t.Fatalf("should have panicked!")
+ }
+ if got, want := e.(error), fmt.Errorf("arrow/ipc: duplicate id=%d", 0); got.Error() != want.Error() {
+ t.Fatalf("invalid panic message.\ngot= %q\nwant=%q", got, want)
+ }
+ }()
+
+ memo := newMemo()
+ defer memo.delete()
+
+ if got, want := memo.Len(), 0; got != want {
+ t.Fatalf("invalid length: got=%d, want=%d", got, want)
+ }
+
+ memo.Add(tc.ids[0], tc.vs[0])
+ memo.Add(tc.ids[1], tc.vs[1])
+ })
+ }
+}
diff --git a/go/arrow/ipc/feather/fbs/CTable.go b/go/arrow/ipc/feather/fbs/CTable.go
new file mode 100644
index 0000000..09d262f
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/CTable.go
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type CTable struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsCTable(buf []byte, offset flatbuffers.UOffsetT) *CTable {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &CTable{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *CTable) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *CTable) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// Some text (or a name) metadata about what the file is, optional
+func (rcv *CTable) Description() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+/// Some text (or a name) metadata about what the file is, optional
+func (rcv *CTable) NumRows() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *CTable) MutateNumRows(n int64) bool {
+ return rcv._tab.MutateInt64Slot(6, n)
+}
+
+func (rcv *CTable) Columns(obj *Column, j int) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ x := rcv._tab.Vector(o)
+ x += flatbuffers.UOffsetT(j) * 4
+ x = rcv._tab.Indirect(x)
+ obj.Init(rcv._tab.Bytes, x)
+ return true
+ }
+ return false
+}
+
+func (rcv *CTable) ColumnsLength() int {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.VectorLen(o)
+ }
+ return 0
+}
+
+/// Version number of the Feather format
+func (rcv *CTable) Version() int32 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ return rcv._tab.GetInt32(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Version number of the Feather format
+func (rcv *CTable) MutateVersion(n int32) bool {
+ return rcv._tab.MutateInt32Slot(10, n)
+}
+
+/// Table metadata (likely JSON), not yet used
+func (rcv *CTable) Metadata() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+/// Table metadata (likely JSON), not yet used
+func CTableStart(builder *flatbuffers.Builder) {
+ builder.StartObject(5)
+}
+func CTableAddDescription(builder *flatbuffers.Builder, description flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(description), 0)
+}
+func CTableAddNumRows(builder *flatbuffers.Builder, numRows int64) {
+ builder.PrependInt64Slot(1, numRows, 0)
+}
+func CTableAddColumns(builder *flatbuffers.Builder, columns flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(columns), 0)
+}
+func CTableStartColumnsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT {
+ return builder.StartVector(4, numElems, 4)
+}
+func CTableAddVersion(builder *flatbuffers.Builder, version int32) {
+ builder.PrependInt32Slot(3, version, 0)
+}
+func CTableAddMetadata(builder *flatbuffers.Builder, metadata flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(metadata), 0)
+}
+func CTableEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/ipc/feather/fbs/CategoryMetadata.go b/go/arrow/ipc/feather/fbs/CategoryMetadata.go
new file mode 100644
index 0000000..e2b56f2
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/CategoryMetadata.go
@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type CategoryMetadata struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsCategoryMetadata(buf []byte, offset flatbuffers.UOffsetT) *CategoryMetadata {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &CategoryMetadata{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *CategoryMetadata) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *CategoryMetadata) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+/// The category codes are presumed to be integers that are valid indexes into
+/// the levels array
+func (rcv *CategoryMetadata) Levels(obj *PrimitiveArray) *PrimitiveArray {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ x := rcv._tab.Indirect(o + rcv._tab.Pos)
+ if obj == nil {
+ obj = new(PrimitiveArray)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+/// The category codes are presumed to be integers that are valid indexes into
+/// the levels array
+func (rcv *CategoryMetadata) Ordered() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *CategoryMetadata) MutateOrdered(n byte) bool {
+ return rcv._tab.MutateByteSlot(6, n)
+}
+
+func CategoryMetadataStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func CategoryMetadataAddLevels(builder *flatbuffers.Builder, levels flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(levels), 0)
+}
+func CategoryMetadataAddOrdered(builder *flatbuffers.Builder, ordered byte) {
+ builder.PrependByteSlot(1, ordered, 0)
+}
+func CategoryMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/ipc/feather/fbs/Column.go b/go/arrow/ipc/feather/fbs/Column.go
new file mode 100644
index 0000000..275ce4c
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/Column.go
@@ -0,0 +1,117 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Column struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsColumn(buf []byte, offset flatbuffers.UOffsetT) *Column {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &Column{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *Column) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *Column) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *Column) Name() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+func (rcv *Column) Values(obj *PrimitiveArray) *PrimitiveArray {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ x := rcv._tab.Indirect(o + rcv._tab.Pos)
+ if obj == nil {
+ obj = new(PrimitiveArray)
+ }
+ obj.Init(rcv._tab.Bytes, x)
+ return obj
+ }
+ return nil
+}
+
+func (rcv *Column) MetadataType() byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.GetByte(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *Column) MutateMetadataType(n byte) bool {
+ return rcv._tab.MutateByteSlot(8, n)
+}
+
+func (rcv *Column) Metadata(obj *flatbuffers.Table) bool {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ rcv._tab.Union(obj, o)
+ return true
+ }
+ return false
+}
+
+/// This should (probably) be JSON
+func (rcv *Column) UserMetadata() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+/// This should (probably) be JSON
+func ColumnStart(builder *flatbuffers.Builder) {
+ builder.StartObject(5)
+}
+func ColumnAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0)
+}
+func ColumnAddValues(builder *flatbuffers.Builder, values flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(values), 0)
+}
+func ColumnAddMetadataType(builder *flatbuffers.Builder, metadataType byte) {
+ builder.PrependByteSlot(2, metadataType, 0)
+}
+func ColumnAddMetadata(builder *flatbuffers.Builder, metadata flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(metadata), 0)
+}
+func ColumnAddUserMetadata(builder *flatbuffers.Builder, userMetadata flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(userMetadata), 0)
+}
+func ColumnEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/ipc/feather/fbs/DateMetadata.go
similarity index 51%
copy from go/arrow/go.mod
copy to go/arrow/ipc/feather/fbs/DateMetadata.go
index 451f68e..bbe0523 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/ipc/feather/fbs/DateMetadata.go
@@ -14,11 +14,37 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
)
+
+type DateMetadata struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsDateMetadata(buf []byte, offset flatbuffers.UOffsetT) *DateMetadata {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &DateMetadata{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *DateMetadata) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *DateMetadata) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func DateMetadataStart(builder *flatbuffers.Builder) {
+ builder.StartObject(0)
+}
+func DateMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/ipc/feather/fbs/Encoding.go
similarity index 60%
copy from go/arrow/go.mod
copy to go/arrow/ipc/feather/fbs/Encoding.go
index 451f68e..7cf3679 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/ipc/feather/fbs/Encoding.go
@@ -14,11 +14,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package fbs
+
+type Encoding = int8
+const (
+ EncodingPLAIN Encoding = 0
+ /// Data is stored dictionary-encoded
+ /// dictionary size: <INT32 Dictionary size>
+ /// dictionary data: <TYPE primitive array>
+ /// dictionary index: <INT32 primitive array>
+ ///
+ /// TODO: do we care about storing the index values in a smaller typeclass
+ EncodingDICTIONARY Encoding = 1
)
+
+var EnumNamesEncoding = map[Encoding]string{
+ EncodingPLAIN:"PLAIN",
+ EncodingDICTIONARY:"DICTIONARY",
+}
+
diff --git a/go/arrow/ipc/feather/fbs/PrimitiveArray.go b/go/arrow/ipc/feather/fbs/PrimitiveArray.go
new file mode 100644
index 0000000..18e81b4
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/PrimitiveArray.go
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type PrimitiveArray struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsPrimitiveArray(buf []byte, offset flatbuffers.UOffsetT) *PrimitiveArray {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &PrimitiveArray{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *PrimitiveArray) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *PrimitiveArray) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *PrimitiveArray) Type() int8 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt8(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *PrimitiveArray) MutateType(n int8) bool {
+ return rcv._tab.MutateInt8Slot(4, n)
+}
+
+func (rcv *PrimitiveArray) Encoding() int8 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.GetInt8(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *PrimitiveArray) MutateEncoding(n int8) bool {
+ return rcv._tab.MutateInt8Slot(6, n)
+}
+
+/// Relative memory offset of the start of the array data excluding the size
+/// of the metadata
+func (rcv *PrimitiveArray) Offset() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(8))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// Relative memory offset of the start of the array data excluding the size
+/// of the metadata
+func (rcv *PrimitiveArray) MutateOffset(n int64) bool {
+ return rcv._tab.MutateInt64Slot(8, n)
+}
+
+/// The number of logical values in the array
+func (rcv *PrimitiveArray) Length() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(10))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// The number of logical values in the array
+func (rcv *PrimitiveArray) MutateLength(n int64) bool {
+ return rcv._tab.MutateInt64Slot(10, n)
+}
+
+/// The number of observed nulls
+func (rcv *PrimitiveArray) NullCount() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(12))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// The number of observed nulls
+func (rcv *PrimitiveArray) MutateNullCount(n int64) bool {
+ return rcv._tab.MutateInt64Slot(12, n)
+}
+
+/// The total size of the actual data in the file
+func (rcv *PrimitiveArray) TotalBytes() int64 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(14))
+ if o != 0 {
+ return rcv._tab.GetInt64(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+/// The total size of the actual data in the file
+func (rcv *PrimitiveArray) MutateTotalBytes(n int64) bool {
+ return rcv._tab.MutateInt64Slot(14, n)
+}
+
+func PrimitiveArrayStart(builder *flatbuffers.Builder) {
+ builder.StartObject(6)
+}
+func PrimitiveArrayAddType(builder *flatbuffers.Builder, type_ int8) {
+ builder.PrependInt8Slot(0, type_, 0)
+}
+func PrimitiveArrayAddEncoding(builder *flatbuffers.Builder, encoding int8) {
+ builder.PrependInt8Slot(1, encoding, 0)
+}
+func PrimitiveArrayAddOffset(builder *flatbuffers.Builder, offset int64) {
+ builder.PrependInt64Slot(2, offset, 0)
+}
+func PrimitiveArrayAddLength(builder *flatbuffers.Builder, length int64) {
+ builder.PrependInt64Slot(3, length, 0)
+}
+func PrimitiveArrayAddNullCount(builder *flatbuffers.Builder, nullCount int64) {
+ builder.PrependInt64Slot(4, nullCount, 0)
+}
+func PrimitiveArrayAddTotalBytes(builder *flatbuffers.Builder, totalBytes int64) {
+ builder.PrependInt64Slot(5, totalBytes, 0)
+}
+func PrimitiveArrayEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/ipc/feather/fbs/TimeMetadata.go b/go/arrow/ipc/feather/fbs/TimeMetadata.go
new file mode 100644
index 0000000..63408ae
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/TimeMetadata.go
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type TimeMetadata struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsTimeMetadata(buf []byte, offset flatbuffers.UOffsetT) *TimeMetadata {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &TimeMetadata{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *TimeMetadata) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *TimeMetadata) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *TimeMetadata) Unit() int8 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt8(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *TimeMetadata) MutateUnit(n int8) bool {
+ return rcv._tab.MutateInt8Slot(4, n)
+}
+
+func TimeMetadataStart(builder *flatbuffers.Builder) {
+ builder.StartObject(1)
+}
+func TimeMetadataAddUnit(builder *flatbuffers.Builder, unit int8) {
+ builder.PrependInt8Slot(0, unit, 0)
+}
+func TimeMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/go.mod b/go/arrow/ipc/feather/fbs/TimeUnit.go
similarity index 65%
copy from go/arrow/go.mod
copy to go/arrow/ipc/feather/fbs/TimeUnit.go
index 451f68e..afd3876 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/ipc/feather/fbs/TimeUnit.go
@@ -14,11 +14,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package fbs
+
+type TimeUnit = int8
+const (
+ TimeUnitSECOND TimeUnit = 0
+ TimeUnitMILLISECOND TimeUnit = 1
+ TimeUnitMICROSECOND TimeUnit = 2
+ TimeUnitNANOSECOND TimeUnit = 3
)
+
+var EnumNamesTimeUnit = map[TimeUnit]string{
+ TimeUnitSECOND:"SECOND",
+ TimeUnitMILLISECOND:"MILLISECOND",
+ TimeUnitMICROSECOND:"MICROSECOND",
+ TimeUnitNANOSECOND:"NANOSECOND",
+}
+
diff --git a/go/arrow/ipc/feather/fbs/TimestampMetadata.go b/go/arrow/ipc/feather/fbs/TimestampMetadata.go
new file mode 100644
index 0000000..86ac293
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/TimestampMetadata.go
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+ flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type TimestampMetadata struct {
+ _tab flatbuffers.Table
+}
+
+func GetRootAsTimestampMetadata(buf []byte, offset flatbuffers.UOffsetT) *TimestampMetadata {
+ n := flatbuffers.GetUOffsetT(buf[offset:])
+ x := &TimestampMetadata{}
+ x.Init(buf, n+offset)
+ return x
+}
+
+func (rcv *TimestampMetadata) Init(buf []byte, i flatbuffers.UOffsetT) {
+ rcv._tab.Bytes = buf
+ rcv._tab.Pos = i
+}
+
+func (rcv *TimestampMetadata) Table() flatbuffers.Table {
+ return rcv._tab
+}
+
+func (rcv *TimestampMetadata) Unit() int8 {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(4))
+ if o != 0 {
+ return rcv._tab.GetInt8(o + rcv._tab.Pos)
+ }
+ return 0
+}
+
+func (rcv *TimestampMetadata) MutateUnit(n int8) bool {
+ return rcv._tab.MutateInt8Slot(4, n)
+}
+
+/// Timestamp data is assumed to be UTC, but the time zone is stored here for
+/// presentation as localized
+func (rcv *TimestampMetadata) Timezone() []byte {
+ o := flatbuffers.UOffsetT(rcv._tab.Offset(6))
+ if o != 0 {
+ return rcv._tab.ByteVector(o + rcv._tab.Pos)
+ }
+ return nil
+}
+
+/// Timestamp data is assumed to be UTC, but the time zone is stored here for
+/// presentation as localized
+func TimestampMetadataStart(builder *flatbuffers.Builder) {
+ builder.StartObject(2)
+}
+func TimestampMetadataAddUnit(builder *flatbuffers.Builder, unit int8) {
+ builder.PrependInt8Slot(0, unit, 0)
+}
+func TimestampMetadataAddTimezone(builder *flatbuffers.Builder, timezone flatbuffers.UOffsetT) {
+ builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(timezone), 0)
+}
+func TimestampMetadataEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+ return builder.EndObject()
+}
diff --git a/go/arrow/ipc/feather/fbs/Type.go b/go/arrow/ipc/feather/fbs/Type.go
new file mode 100644
index 0000000..f6a6771
--- /dev/null
+++ b/go/arrow/ipc/feather/fbs/Type.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+/// Feather is an experimental serialization format implemented using
+/// techniques from Apache Arrow. It was created as a proof-of-concept of an
+/// interoperable file format for storing data frames originating in Python or
+/// R. It enabled the developers to sidestep some of the open design questions
+/// in Arrow from early 2016 and instead create something simple and useful for
+/// the intended use cases.
+type Type = int8
+const (
+ TypeBOOL Type = 0
+ TypeINT8 Type = 1
+ TypeINT16 Type = 2
+ TypeINT32 Type = 3
+ TypeINT64 Type = 4
+ TypeUINT8 Type = 5
+ TypeUINT16 Type = 6
+ TypeUINT32 Type = 7
+ TypeUINT64 Type = 8
+ TypeFLOAT Type = 9
+ TypeDOUBLE Type = 10
+ TypeUTF8 Type = 11
+ TypeBINARY Type = 12
+ TypeCATEGORY Type = 13
+ TypeTIMESTAMP Type = 14
+ TypeDATE Type = 15
+ TypeTIME Type = 16
+)
+
+var EnumNamesType = map[Type]string{
+ TypeBOOL:"BOOL",
+ TypeINT8:"INT8",
+ TypeINT16:"INT16",
+ TypeINT32:"INT32",
+ TypeINT64:"INT64",
+ TypeUINT8:"UINT8",
+ TypeUINT16:"UINT16",
+ TypeUINT32:"UINT32",
+ TypeUINT64:"UINT64",
+ TypeFLOAT:"FLOAT",
+ TypeDOUBLE:"DOUBLE",
+ TypeUTF8:"UTF8",
+ TypeBINARY:"BINARY",
+ TypeCATEGORY:"CATEGORY",
+ TypeTIMESTAMP:"TIMESTAMP",
+ TypeDATE:"DATE",
+ TypeTIME:"TIME",
+}
+
diff --git a/go/arrow/go.mod b/go/arrow/ipc/feather/fbs/TypeMetadata.go
similarity index 57%
copy from go/arrow/go.mod
copy to go/arrow/ipc/feather/fbs/TypeMetadata.go
index 451f68e..b28e9f1 100644
--- a/go/arrow/go.mod
+++ b/go/arrow/ipc/feather/fbs/TypeMetadata.go
@@ -14,11 +14,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-module github.com/apache/arrow/go/arrow
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
-require (
- github.com/davecgh/go-spew v1.1.0 // indirect
- github.com/pkg/errors v0.8.1
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/stretchr/testify v1.2.0
+package fbs
+
+type TypeMetadata = byte
+const (
+ TypeMetadataNONE TypeMetadata = 0
+ TypeMetadataCategoryMetadata TypeMetadata = 1
+ TypeMetadataTimestampMetadata TypeMetadata = 2
+ TypeMetadataDateMetadata TypeMetadata = 3
+ TypeMetadataTimeMetadata TypeMetadata = 4
)
+
+var EnumNamesTypeMetadata = map[TypeMetadata]string{
+ TypeMetadataNONE:"NONE",
+ TypeMetadataCategoryMetadata:"CategoryMetadata",
+ TypeMetadataTimestampMetadata:"TimestampMetadata",
+ TypeMetadataDateMetadata:"DateMetadata",
+ TypeMetadataTimeMetadata:"TimeMetadata",
+}
+
diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go
new file mode 100644
index 0000000..4f8646e
--- /dev/null
+++ b/go/arrow/ipc/file_reader.go
@@ -0,0 +1,453 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "io"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/array"
+ "github.com/apache/arrow/go/arrow/internal/bitutil"
+ "github.com/apache/arrow/go/arrow/internal/flatbuf"
+ "github.com/apache/arrow/go/arrow/memory"
+ "github.com/pkg/errors"
+)
+
+// FileReader is an Arrow file reader.
+type FileReader struct {
+ r ReadAtSeeker
+
+ footer struct {
+ offset int64
+ buffer *memory.Buffer
+ data *flatbuf.Footer
+ }
+
+ fields dictTypeMap
+ memo dictMemo
+
+ schema *arrow.Schema
+ record array.Record
+}
+
+// NewFileReader opens an Arrow file using the provided reader r.
+func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error) {
+ var (
+ f = FileReader{
+ r: r,
+ fields: make(dictTypeMap),
+ memo: newMemo(),
+ }
+ cfg = newConfig()
+ err error
+ )
+
+ for _, opt := range opts {
+ opt(cfg)
+ }
+
+ if cfg.footer.offset <= 0 {
+ cfg.footer.offset, err = f.r.Seek(0, io.SeekEnd)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could retrieve footer offset")
+ }
+ }
+ f.footer.offset = cfg.footer.offset
+
+ err = f.readFooter()
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not decode footer")
+ }
+
+ err = f.readSchema()
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not decode schema")
+ }
+
+ return &f, err
+}
+
+func (f *FileReader) readFooter() error {
+ var err error
+
+ if f.footer.offset <= int64(len(Magic)*2+4) {
+ return fmt.Errorf("arrow/ipc: file too small (%d)", f.footer.offset)
+ }
+
+ eof := int64(len(Magic) + 4)
+ buf := make([]byte, eof)
+ n, err := f.r.ReadAt(buf, f.footer.offset-eof)
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could not read footer")
+ }
+ if n != len(buf) {
+ return errors.Errorf("arrow/ipc: could not read %d bytes from end of file", len(buf))
+ }
+
+ if !bytes.Equal(buf[4:], Magic) {
+ return errNotArrowFile
+ }
+
+ size := int64(binary.LittleEndian.Uint32(buf[:4]))
+ if size <= 0 || size+int64(len(Magic)*2+4) > f.footer.offset {
+ return errInconsistentFileMetadata
+ }
+
+ buf = make([]byte, size)
+ n, err = f.r.ReadAt(buf, f.footer.offset-size-eof)
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could not read footer data")
+ }
+ if n != len(buf) {
+ return errors.Errorf("arrow/ipc: could not read %d bytes from footer data", len(buf))
+ }
+
+ f.footer.buffer = memory.NewBufferBytes(buf)
+ f.footer.data = flatbuf.GetRootAsFooter(buf, 0)
+ return err
+}
+
+func (f *FileReader) readSchema() error {
+ var err error
+ f.fields, err = dictTypesFromFB(f.footer.data.Schema(nil))
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could not load dictionary types from file")
+ }
+
+ for i := 0; i < f.NumDictionaries(); i++ {
+ blk, err := f.dict(i)
+ if err != nil {
+ return errors.Wrapf(err, "arrow/ipc: could read dictionary[%d]", i)
+ }
+ switch {
+ case !bitutil.IsMultipleOf8(blk.Offset):
+ return errors.Errorf("arrow/ipc: invalid file offset=%d for dictionary %d", blk.Offset, i)
+ case !bitutil.IsMultipleOf8(int64(blk.Meta)):
+ return errors.Errorf("arrow/ipc: invalid file metadata=%d position for dictionary %d", blk.Meta, i)
+ case !bitutil.IsMultipleOf8(blk.Body):
+ return errors.Errorf("arrow/ipc: invalid file body=%d position for dictionary %d", blk.Body, i)
+ }
+
+ msg, err := blk.NewMessage()
+ if err != nil {
+ return err
+ }
+ defer msg.Release()
+
+ id, dict, err := readDictionary(msg.meta, f.fields, f.r)
+ if err != nil {
+ return errors.Wrapf(err, "arrow/ipc: could not read dictionary %d from file", i)
+ }
+ f.memo.Add(id, dict)
+ dict.Release() // memo.Add increases ref-count of dict.
+ }
+
+ schema := f.footer.data.Schema(nil)
+ if schema == nil {
+ return errors.New("arrow/ipc: could not load schema from flatbuffer data")
+ }
+ f.schema, err = schemaFromFB(schema, &f.memo)
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could not read schema")
+ }
+
+ return err
+}
+
+func (f *FileReader) block(i int) (fileBlock, error) {
+ var blk flatbuf.Block
+ if !f.footer.data.RecordBatches(&blk, i) {
+ return fileBlock{}, errors.Errorf("arrow/ipc: could not extract file block %d", i)
+ }
+
+ return fileBlock{
+ Offset: blk.Offset(),
+ Meta: blk.MetaDataLength(),
+ Body: blk.BodyLength(),
+ r: f.r,
+ }, nil
+}
+
+func (f *FileReader) dict(i int) (fileBlock, error) {
+ var blk flatbuf.Block
+ if !f.footer.data.Dictionaries(&blk, i) {
+ return fileBlock{}, errors.Errorf("arrow/ipc: could not extract dictionary block %d", i)
+ }
+
+ return fileBlock{
+ Offset: blk.Offset(),
+ Meta: blk.MetaDataLength(),
+ Body: blk.BodyLength(),
+ r: f.r,
+ }, nil
+}
+
+func (f *FileReader) Schema() *arrow.Schema {
+ return f.schema
+}
+
+func (f *FileReader) NumDictionaries() int {
+ if f.footer.data == nil {
+ return 0
+ }
+ return f.footer.data.DictionariesLength()
+}
+
+func (f *FileReader) NumRecords() int {
+ return f.footer.data.RecordBatchesLength()
+}
+
+func (f *FileReader) Version() MetadataVersion {
+ return MetadataVersion(f.footer.data.Version())
+}
+
+// Close cleans up resources used by the File.
+// Close does not close the underlying reader.
+func (f *FileReader) Close() error {
+ if f.footer.data != nil {
+ f.footer.data = nil
+ }
+
+ if f.footer.buffer != nil {
+ f.footer.buffer.Release()
+ f.footer.buffer = nil
+ }
+
+ if f.record != nil {
+ f.record.Release()
+ f.record = nil
+ }
+ return nil
+}
+
+// Record returns the i-th record from the file.
+// The returned value is valid until the next call to Record.
+// Users need to call Retain on that Record to keep it valid for longer.
+func (f *FileReader) Record(i int) (array.Record, error) {
+ if i < 0 || i > f.NumRecords() {
+ panic("arrow/ipc: record index out of bounds")
+ }
+
+ blk, err := f.block(i)
+ if err != nil {
+ return nil, err
+ }
+ switch {
+ case !bitutil.IsMultipleOf8(blk.Offset):
+ return nil, errors.Errorf("arrow/ipc: invalid file offset=%d for record %d", blk.Offset, i)
+ case !bitutil.IsMultipleOf8(int64(blk.Meta)):
+ return nil, errors.Errorf("arrow/ipc: invalid file metadata=%d position for record %d", blk.Meta, i)
+ case !bitutil.IsMultipleOf8(blk.Body):
+ return nil, errors.Errorf("arrow/ipc: invalid file body=%d position for record %d", blk.Body, i)
+ }
+
+ msg, err := blk.NewMessage()
+ if err != nil {
+ return nil, err
+ }
+ defer msg.Release()
+
+ if msg.Type() != MessageRecordBatch {
+ return nil, errors.Errorf("arrow/ipc: message %d is not a Record", i)
+ }
+
+ if f.record != nil {
+ f.record.Release()
+ }
+
+ f.record = newRecord(f.schema, msg.meta, bytes.NewReader(msg.body.Bytes()))
+ return f.record, nil
+}
+
+func newRecord(schema *arrow.Schema, meta *memory.Buffer, body ReadAtSeeker) array.Record {
+ var (
+ msg = flatbuf.GetRootAsMessage(meta.Bytes(), 0)
+ md flatbuf.RecordBatch
+ )
+ initFB(&md, msg.Header)
+ rows := md.Length()
+
+ ctx := &arrayLoaderContext{
+ src: ipcSource{
+ meta: &md,
+ r: body,
+ },
+ max: kMaxNestingDepth,
+ }
+
+ cols := make([]array.Interface, len(schema.Fields()))
+ for i, field := range schema.Fields() {
+ cols[i] = ctx.loadArray(field.Type)
+ }
+
+ return array.NewRecord(schema, cols, rows)
+}
+
+type ipcSource struct {
+ meta *flatbuf.RecordBatch
+ r ReadAtSeeker
+}
+
+func (src *ipcSource) buffer(i int) *memory.Buffer {
+ var buf flatbuf.Buffer
+ if !src.meta.Buffers(&buf, i) {
+ panic("buffer index out of bound")
+ }
+ if buf.Length() == 0 {
+ return memory.NewBufferBytes(nil)
+ }
+
+ raw := make([]byte, buf.Length())
+ _, err := src.r.ReadAt(raw, buf.Offset())
+ if err != nil {
+ panic(err)
+ }
+
+ return memory.NewBufferBytes(raw)
+}
+
+func (src *ipcSource) fieldMetadata(i int) *flatbuf.FieldNode {
+ var node flatbuf.FieldNode
+ if !src.meta.Nodes(&node, i) {
+ panic("field metadata out of bound")
+ }
+ return &node
+}
+
+type arrayLoaderContext struct {
+ src ipcSource
+ ifield int
+ ibuffer int
+ max int
+}
+
+func (ctx *arrayLoaderContext) field() *flatbuf.FieldNode {
+ field := ctx.src.fieldMetadata(ctx.ifield)
+ ctx.ifield++
+ return field
+}
+
+func (ctx *arrayLoaderContext) buffer() *memory.Buffer {
+ buf := ctx.src.buffer(ctx.ibuffer)
+ ctx.ibuffer++
+ return buf
+}
+
+func (ctx *arrayLoaderContext) loadArray(dt arrow.DataType) array.Interface {
+ switch dt := dt.(type) {
+ case *arrow.NullType:
+ return ctx.loadNull()
+
+ case *arrow.BooleanType,
+ *arrow.Int8Type, *arrow.Int16Type, *arrow.Int32Type, *arrow.Int64Type,
+ *arrow.Uint8Type, *arrow.Uint16Type, *arrow.Uint32Type, *arrow.Uint64Type,
+ *arrow.Float32Type, *arrow.Float64Type:
+ return ctx.loadPrimitive(dt)
+
+ default:
+ panic(errors.Errorf("array type %T not handled yet", dt))
+ }
+}
+
+func (ctx *arrayLoaderContext) loadCommon(nbufs int) (*flatbuf.FieldNode, []*memory.Buffer) {
+ buffers := make([]*memory.Buffer, 0, nbufs)
+ field := ctx.field()
+
+ var buf *memory.Buffer
+ switch field.NullCount() {
+ case 0:
+ ctx.ibuffer++
+ default:
+ buf = ctx.buffer()
+ }
+ buffers = append(buffers, buf)
+
+ return field, buffers
+}
+
+func (ctx *arrayLoaderContext) loadNull() array.Interface {
+ field, buffers := ctx.loadCommon(1)
+ buffers = append(buffers, ctx.buffer())
+
+ data := array.NewData(arrow.Null, int(field.Length()), buffers, nil, int(field.NullCount()), 0)
+ defer data.Release()
+
+ return array.MakeFromData(data)
+}
+
+func (ctx *arrayLoaderContext) loadPrimitive(dt arrow.DataType) array.Interface {
+ field, buffers := ctx.loadCommon(2)
+
+ switch field.Length() {
+ case 0:
+ buffers = append(buffers, nil)
+ ctx.ibuffer++
+ default:
+ buffers = append(buffers, ctx.buffer())
+ }
+
+ data := array.NewData(dt, int(field.Length()), buffers, nil, int(field.NullCount()), 0)
+ defer data.Release()
+
+ return array.MakeFromData(data)
+}
+
+func readDictionary(meta *memory.Buffer, types dictTypeMap, r ReadAtSeeker) (int64, array.Interface, error) {
+ // msg := flatbuf.GetRootAsMessage(meta.Bytes(), 0)
+ // var dictBatch flatbuf.DictionaryBatch
+ // initFB(&dictBatch, msg.Header)
+ //
+ // id := dictBatch.Id()
+ // v, ok := types[id]
+ // if !ok {
+ // return id, nil, errors.Errorf("arrow/ipc: no type metadata for dictionary with ID=%d", id)
+ // }
+ //
+ // fields := []arrow.Field{v}
+ //
+ // // we need a schema for the record batch.
+ // schema := arrow.NewSchema(fields, nil)
+ //
+ // // the dictionary is embedded in a record batch with a single column.
+ // recBatch := dictBatch.Data(nil)
+ //
+ // var (
+ // batchMeta *memory.Buffer
+ // body *memory.Buffer
+ // )
+ //
+ //
+ // ctx := &arrayLoaderContext{
+ // src: ipcSource{
+ // meta: &md,
+ // r: bytes.NewReader(body.Bytes()),
+ // },
+ // max: kMaxNestingDepth,
+ // }
+ //
+ // cols := make([]array.Interface, len(schema.Fields()))
+ // for i, field := range schema.Fields() {
+ // cols[i] = ctx.loadArray(field.Type)
+ // }
+ //
+ // batch := array.NewRecord(schema, cols, rows)
+
+ panic("not implemented")
+}
diff --git a/go/arrow/ipc/ipc.go b/go/arrow/ipc/ipc.go
new file mode 100644
index 0000000..fc83e94
--- /dev/null
+++ b/go/arrow/ipc/ipc.go
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "io"
+
+ "github.com/apache/arrow/go/arrow/memory"
+)
+
+const (
+ errNotArrowFile = errString("arrow/ipc: not an Arrow file")
+ errInconsistentFileMetadata = errString("arrow/ipc: file is smaller than indicated metadata size")
+)
+
+type errString string
+
+func (s errString) Error() string {
+ return string(s)
+}
+
+type ReadAtSeeker interface {
+ io.Reader
+ io.Seeker
+ io.ReaderAt
+}
+
+type config struct {
+ alloc memory.Allocator
+ footer struct {
+ offset int64
+ }
+}
+
+func newConfig() *config {
+ return &config{
+ alloc: memory.NewGoAllocator(),
+ }
+}
+
+// Option is a functional option to configure opening or creating Arrow files
+// and streams.
+type Option func(*config)
+
+// WithFooterOffset specifies the Arrow footer position in bytes.
+func WithFooterOffset(offset int64) Option {
+ return func(cfg *config) {
+ cfg.footer.offset = offset
+ }
+}
+
+// WithAllocator specifies the Arrow memory allocator used while building records.
+func WithAllocator(mem memory.Allocator) Option {
+ return func(cfg *config) {
+ cfg.alloc = mem
+ }
+}
diff --git a/go/arrow/ipc/message.go b/go/arrow/ipc/message.go
new file mode 100644
index 0000000..bb12dbb
--- /dev/null
+++ b/go/arrow/ipc/message.go
@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "encoding/binary"
+ "fmt"
+ "io"
+ "sync/atomic"
+
+ "github.com/apache/arrow/go/arrow/internal/debug"
+ "github.com/apache/arrow/go/arrow/internal/flatbuf"
+ "github.com/apache/arrow/go/arrow/memory"
+ "github.com/pkg/errors"
+)
+
+// MetadataVersion represents the Arrow metadata version.
+type MetadataVersion flatbuf.MetadataVersion
+
+const (
+ MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0
+ MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0
+ MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1
+ MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0
+)
+
+func (m MetadataVersion) String() string {
+ if v, ok := flatbuf.EnumNamesMetadataVersion[int16(m)]; ok {
+ return v
+ }
+ return fmt.Sprintf("MetadataVersion(%d)", int16(m))
+}
+
+// MessageType represents the type of Message in an Arrow format.
+type MessageType flatbuf.MessageHeader
+
+const (
+ MessageNone = MessageType(flatbuf.MessageHeaderNONE)
+ MessageSchema = MessageType(flatbuf.MessageHeaderSchema)
+ MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch)
+ MessageRecordBatch = MessageType(flatbuf.MessageHeaderRecordBatch)
+ MessageTensor = MessageType(flatbuf.MessageHeaderTensor)
+ MessageSparseTensor = MessageType(flatbuf.MessageHeaderSparseTensor)
+)
+
+func (m MessageType) String() string {
+ if v, ok := flatbuf.EnumNamesMessageHeader[byte(m)]; ok {
+ return v
+ }
+ return fmt.Sprintf("MessageType(%d)", int(m))
+}
+
+const (
+ // maxNestingDepth is an arbitrary value to catch user mistakes.
+ // For deeply nested schemas, it is expected the user will indicate
+ // explicitly the maximum allowed recursion depth.
+ maxNestingDepth = 64
+)
+
+// Message is an IPC message, including metadata and body.
+type Message struct {
+ refCount int64
+ msg *flatbuf.Message
+ meta *memory.Buffer
+ body *memory.Buffer
+}
+
+// NewMessage creates a new message from the metadata and body buffers.
+// NewMessage panics if any of these buffers is nil.
+func NewMessage(meta, body *memory.Buffer) *Message {
+ if meta == nil || body == nil {
+ panic("arrow/ipc: nil buffers")
+ }
+ meta.Retain()
+ body.Retain()
+ return &Message{
+ refCount: 1,
+ msg: flatbuf.GetRootAsMessage(meta.Bytes(), 0),
+ meta: meta,
+ body: body,
+ }
+}
+
+func newMessageFromFB(meta *flatbuf.Message, body *memory.Buffer) *Message {
+ if meta == nil || body == nil {
+ panic("arrow/ipc: nil buffers")
+ }
+ body.Retain()
+ return &Message{
+ refCount: 1,
+ msg: meta,
+ meta: memory.NewBufferBytes(meta.Table().Bytes),
+ body: body,
+ }
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (msg *Message) Retain() {
+ atomic.AddInt64(&msg.refCount, 1)
+}
+
+// Release decreases the reference count by 1.
+// Release may be called simultaneously from multiple goroutines.
+// When the reference count goes to zero, the memory is freed.
+func (msg *Message) Release() {
+ debug.Assert(atomic.LoadInt64(&msg.refCount) > 0, "too many releases")
+
+ if atomic.AddInt64(&msg.refCount, -1) == 0 {
+ msg.meta.Release()
+ msg.body.Release()
+ msg.msg = nil
+ msg.meta = nil
+ msg.body = nil
+ }
+}
+
+func (msg *Message) Version() MetadataVersion {
+ return MetadataVersion(msg.msg.Version())
+}
+
+func (msg *Message) Type() MessageType {
+ return MessageType(msg.msg.HeaderType())
+}
+
+func (msg *Message) BodyLen() int64 {
+ return msg.msg.BodyLength()
+}
+
+// MessageReader reads messages from an io.Reader.
+type MessageReader struct {
+ r io.Reader
+
+ refCount int64
+ msg *Message
+}
+
+// NewMessageReader returns a reader that reads messages from an input stream.
+func NewMessageReader(r io.Reader) *MessageReader {
+ return &MessageReader{r: r, refCount: 1}
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (r *MessageReader) Retain() {
+ atomic.AddInt64(&r.refCount, 1)
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (r *MessageReader) Release() {
+ debug.Assert(atomic.LoadInt64(&r.refCount) > 0, "too many releases")
+
+ if atomic.AddInt64(&r.refCount, -1) == 0 {
+ if r.msg != nil {
+ r.msg.Release()
+ r.msg = nil
+ }
+ }
+}
+
+// Message returns the current message that has been extracted from the
+// underlying stream.
+// It is valid until the next call to Message.
+func (r *MessageReader) Message() (*Message, error) {
+ var buf = make([]byte, 4)
+ _, err := io.ReadFull(r.r, buf)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not read message length")
+ }
+ msgLen := int32(binary.LittleEndian.Uint32(buf))
+ if msgLen == 0 {
+ // optional 0 EOS control message
+ return nil, io.EOF // FIXME(sbinet): send nil instead? or a special EOS error?
+ }
+
+ buf = make([]byte, msgLen)
+ _, err = io.ReadFull(r.r, buf)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not read message metadata")
+ }
+
+ meta := flatbuf.GetRootAsMessage(buf, 0)
+ bodyLen := meta.BodyLength()
+
+ buf = make([]byte, bodyLen)
+ _, err = io.ReadFull(r.r, buf)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not read message body")
+ }
+ body := memory.NewBufferBytes(buf)
+
+ if r.msg != nil {
+ r.msg.Release()
+ r.msg = nil
+ }
+ r.msg = newMessageFromFB(meta, body)
+
+ return r.msg, nil
+}
diff --git a/go/arrow/ipc/metadata.go b/go/arrow/ipc/metadata.go
new file mode 100644
index 0000000..f3ecc2e
--- /dev/null
+++ b/go/arrow/ipc/metadata.go
@@ -0,0 +1,412 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "fmt"
+ "io"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/internal/flatbuf"
+ "github.com/apache/arrow/go/arrow/memory"
+ flatbuffers "github.com/google/flatbuffers/go"
+ "github.com/pkg/errors"
+)
+
+// Magic string identifying an Apache Arrow file.
+var Magic = []byte("ARROW1")
+
+const (
+ currentMetadataVersion = MetadataV4
+ minMetadataVersion = MetadataV4
+
+ kExtensionTypeKeyName = "arrow_extension_name"
+ kExtensionDataKeyName = "arrow_extension_data"
+
+ // ARROW-109: We set this number arbitrarily to help catch user mistakes. For
+ // deeply nested schemas, it is expected the user will indicate explicitly the
+ // maximum allowed recursion depth
+ kMaxNestingDepth = 64
+)
+
+type fieldMetadata struct {
+ Len int64
+ Nulls int64
+ Offset int64
+}
+
+type bufferMetadata struct {
+ Offset int64 // relative offset into the memory page to the starting byte of the buffer
+ Len int64 // absolute length in bytes of the buffer
+}
+
+type fileBlock struct {
+ Offset int64
+ Meta int32
+ Body int64
+
+ r io.ReaderAt
+}
+
+func (blk fileBlock) NewMessage() (*Message, error) {
+ var (
+ err error
+ buf []byte
+ r = blk.section()
+ )
+
+ buf = make([]byte, blk.Meta)
+ _, err = io.ReadFull(r, buf)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not read message metadata")
+ }
+ meta := memory.NewBufferBytes(buf[4:]) // drop buf-size already known from blk.Meta
+
+ buf = make([]byte, blk.Body)
+ _, err = io.ReadFull(r, buf)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not read message body")
+ }
+ body := memory.NewBufferBytes(buf)
+
+ return NewMessage(meta, body), nil
+}
+
+func (blk fileBlock) section() io.Reader {
+ return io.NewSectionReader(blk.r, blk.Offset, int64(blk.Meta)+blk.Body)
+}
+
+func nullableFromFB(v byte) bool {
+ return v != 0
+}
+
+// initFB is a helper function to handle flatbuffers' polymorphism.
+func initFB(t interface {
+ Table() flatbuffers.Table
+ Init([]byte, flatbuffers.UOffsetT)
+}, f func(tbl *flatbuffers.Table) bool) {
+ tbl := t.Table()
+ if !f(&tbl) {
+ panic(errors.Errorf("arrow/ipc: could not initialize %T from flatbuffer", t))
+ }
+ t.Init(tbl.Bytes, tbl.Pos)
+}
+
+func fieldFromFB(field *flatbuf.Field, memo *dictMemo) (arrow.Field, error) {
+ var (
+ err error
+ o arrow.Field
+ )
+
+ o.Name = string(field.Name())
+ o.Nullable = nullableFromFB(field.Nullable())
+ o.Metadata, err = metadataFrom(field)
+ if err != nil {
+ return o, err
+ }
+
+ encoding := field.Dictionary(nil)
+ switch encoding {
+ case nil:
+ n := field.ChildrenLength()
+ children := make([]arrow.Field, n)
+ for i := range children {
+ var childFB flatbuf.Field
+ if !field.Children(&childFB, i) {
+ return o, errors.Errorf("arrow/ipc: could not load field child %d", i)
+ }
+ child, err := fieldFromFB(&childFB, memo)
+ if err != nil {
+ return o, errors.Wrapf(err, "arrow/ipc: could not convert field child %d", i)
+ }
+ children[i] = child
+ }
+
+ o.Type, err = typeFromFB(field, children, o.Metadata)
+ if err != nil {
+ return o, errors.Wrapf(err, "arrow/ipc: could not convert field type")
+ }
+ default:
+ // log.Printf("encoding: %v", encoding.Id())
+ // n := field.ChildrenLength()
+ // log.Printf("children: %v", n)
+ panic("not implemented") // FIXME(sbinet)
+ }
+
+ return o, nil
+}
+
+func fieldFromFBDict(field *flatbuf.Field) (arrow.Field, error) {
+ var (
+ o = arrow.Field{
+ Name: string(field.Name()),
+ Nullable: nullableFromFB(field.Nullable()),
+ }
+ err error
+ memo = newMemo()
+ )
+
+ // any DictionaryEncoding set is ignored here.
+
+ kids := make([]arrow.Field, field.ChildrenLength())
+ for i := range kids {
+ var kid flatbuf.Field
+ if !field.Children(&kid, i) {
+ return o, errors.Errorf("arrow/ipc: could not load field child %d", i)
+ }
+ kids[i], err = fieldFromFB(&kid, &memo)
+ if err != nil {
+ return o, errors.Wrap(err, "arrow/ipc: field from dict")
+ }
+ }
+
+ meta, err := metadataFrom(field)
+ if err != nil {
+ return o, errors.Wrap(err, "arrow/ipc: metadata for field from dict")
+ }
+
+ o.Type, err = typeFromFB(field, kids, meta)
+ if err != nil {
+ return o, errors.Wrap(err, "arrow/ipc: type for field from dict")
+ }
+
+ return o, nil
+}
+
+func typeFromFB(field *flatbuf.Field, children []arrow.Field, md arrow.Metadata) (arrow.DataType, error) {
+ var data flatbuffers.Table
+ if !field.Type(&data) {
+ return nil, errors.Errorf("arrow/ipc: could not load field type data")
+ }
+
+ dt, err := concreteTypeFromFB(field.TypeType(), data, children)
+ if err != nil {
+ return dt, err
+ }
+
+ // look for extension metadata in custom metadata field.
+ if md.Len() > 0 {
+ i := md.FindKey(kExtensionTypeKeyName)
+ if i < 0 {
+ return dt, err
+ }
+
+ panic("not implemented") // FIXME(sbinet)
+ }
+
+ return dt, err
+}
+
+func concreteTypeFromFB(typ flatbuf.Type, data flatbuffers.Table, children []arrow.Field) (arrow.DataType, error) {
+ var (
+ dt arrow.DataType
+ err error
+ )
+
+ switch typ {
+ case flatbuf.TypeNONE:
+ return nil, errors.Errorf("arrow/ipc: Type metadata cannot be none")
+
+ case flatbuf.TypeNull:
+ return arrow.Null, nil
+
+ case flatbuf.TypeInt:
+ var dt flatbuf.Int
+ dt.Init(data.Bytes, data.Pos)
+ return intFromFB(dt)
+
+ case flatbuf.TypeFloatingPoint:
+ var dt flatbuf.FloatingPoint
+ dt.Init(data.Bytes, data.Pos)
+ return floatFromFB(dt)
+
+ case flatbuf.TypeBinary:
+ return arrow.BinaryTypes.Binary, nil
+
+ case flatbuf.TypeFixedSizeBinary:
+ var dt flatbuf.FixedSizeBinary
+ dt.Init(data.Bytes, data.Pos)
+ return &arrow.FixedSizeBinaryType{ByteWidth: int(dt.ByteWidth())}, nil
+
+ case flatbuf.TypeUtf8:
+ return arrow.BinaryTypes.String, nil
+
+ case flatbuf.TypeBool:
+ return arrow.FixedWidthTypes.Boolean, nil
+
+ default:
+ // FIXME(sbinet): implement all the other types.
+ panic(fmt.Errorf("arrow/ipc: type %v not implemented", flatbuf.EnumNamesType[typ]))
+ }
+
+ return dt, err
+}
+
+func intFromFB(data flatbuf.Int) (arrow.DataType, error) {
+ bw := data.BitWidth()
+ if bw > 64 {
+ return nil, errors.Errorf("arrow/ipc: integers with more than 64 bits not implemented (bits=%d)", bw)
+ }
+ if bw < 8 {
+ return nil, errors.Errorf("arrow/ipc: integers with less than 8 bits not implemented (bits=%d)", bw)
+ }
+
+ switch bw {
+ case 8:
+ switch data.IsSigned() {
+ case 0:
+ return arrow.PrimitiveTypes.Uint8, nil
+ default:
+ return arrow.PrimitiveTypes.Int8, nil
+ }
+
+ case 16:
+ switch data.IsSigned() {
+ case 0:
+ return arrow.PrimitiveTypes.Uint16, nil
+ default:
+ return arrow.PrimitiveTypes.Int16, nil
+ }
+
+ case 32:
+ switch data.IsSigned() {
+ case 0:
+ return arrow.PrimitiveTypes.Uint32, nil
+ default:
+ return arrow.PrimitiveTypes.Int32, nil
+ }
+
+ case 64:
+ switch data.IsSigned() {
+ case 0:
+ return arrow.PrimitiveTypes.Uint64, nil
+ default:
+ return arrow.PrimitiveTypes.Int64, nil
+ }
+ default:
+ return nil, errors.Errorf("arrow/ipc: integers not in cstdint are not implemented")
+ }
+}
+
+func floatFromFB(data flatbuf.FloatingPoint) (arrow.DataType, error) {
+ switch p := data.Precision(); p {
+ case flatbuf.PrecisionHALF:
+ return nil, errors.Errorf("arrow/ipc: float16 not implemented")
+ case flatbuf.PrecisionSINGLE:
+ return arrow.PrimitiveTypes.Float32, nil
+ case flatbuf.PrecisionDOUBLE:
+ return arrow.PrimitiveTypes.Float64, nil
+ default:
+ return nil, errors.Errorf("arrow/ipc: floating point type with %d precision not implemented", p)
+ }
+}
+
+type customMetadataer interface {
+ CustomMetadataLength() int
+ CustomMetadata(*flatbuf.KeyValue, int) bool
+}
+
+func metadataFrom(md customMetadataer) (arrow.Metadata, error) {
+ var (
+ keys = make([]string, md.CustomMetadataLength())
+ vals = make([]string, md.CustomMetadataLength())
+ )
+
+ for i := range keys {
+ var kv flatbuf.KeyValue
+ if !md.CustomMetadata(&kv, i) {
+ return arrow.Metadata{}, errors.Errorf("arrow/ipc: could not read key-value %d from flatbuffer", i)
+ }
+ keys[i] = string(kv.Key())
+ vals[i] = string(kv.Value())
+ }
+
+ return arrow.NewMetadata(keys, vals), nil
+}
+
+func schemaFromFB(schema *flatbuf.Schema, memo *dictMemo) (*arrow.Schema, error) {
+ var (
+ err error
+ fields = make([]arrow.Field, schema.FieldsLength())
+ )
+
+ for i := range fields {
+ var field flatbuf.Field
+ if !schema.Fields(&field, i) {
+ return nil, errors.Errorf("arrow/ipc: could not read field %d from schema", i)
+ }
+
+ fields[i], err = fieldFromFB(&field, memo)
+ if err != nil {
+ return nil, errors.Wrapf(err, "arrow/ipc: could not convert field %d from flatbuf", i)
+ }
+ }
+
+ md, err := metadataFrom(schema)
+ if err != nil {
+ return nil, errors.Wrapf(err, "arrow/ipc: could not convert schema metadata from flatbuf")
+ }
+
+ return arrow.NewSchema(fields, &md), nil
+}
+
+func dictTypesFromFB(schema *flatbuf.Schema) (dictTypeMap, error) {
+ var (
+ err error
+ fields = make(dictTypeMap, schema.FieldsLength())
+ )
+ for i := 0; i < schema.FieldsLength(); i++ {
+ var field flatbuf.Field
+ if !schema.Fields(&field, i) {
+ return nil, errors.Errorf("arrow/ipc: could not load field %d from schema", i)
+ }
+ fields, err = visitField(&field, fields)
+ if err != nil {
+ return nil, errors.Wrapf(err, "arrow/ipc: could not visit field %d from schema", i)
+ }
+ }
+ return fields, err
+}
+
+func visitField(field *flatbuf.Field, dict dictTypeMap) (dictTypeMap, error) {
+ var err error
+ meta := field.Dictionary(nil)
+ switch meta {
+ case nil:
+ // field is not dictionary encoded.
+ // => visit children.
+ for i := 0; i < field.ChildrenLength(); i++ {
+ var child flatbuf.Field
+ if !field.Children(&child, i) {
+ return nil, errors.Errorf("arrow/ipc: could not visit child %d from field", i)
+ }
+ dict, err = visitField(&child, dict)
+ if err != nil {
+ return nil, err
+ }
+ }
+ default:
+ // field is dictionary encoded.
+ // construct the data type for the dictionary: no descendants can be dict-encoded.
+ dfield, err := fieldFromFBDict(field)
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not create data type for dictionary")
+ }
+ dict[meta.Id()] = dfield
+ }
+ return dict, err
+}
diff --git a/go/arrow/ipc/reader.go b/go/arrow/ipc/reader.go
new file mode 100644
index 0000000..dd98f7c
--- /dev/null
+++ b/go/arrow/ipc/reader.go
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipc // import "github.com/apache/arrow/go/arrow/ipc"
+
+import (
+ "bytes"
+ "io"
+ "sync/atomic"
+
+ "github.com/apache/arrow/go/arrow"
+ "github.com/apache/arrow/go/arrow/array"
+ "github.com/apache/arrow/go/arrow/internal/debug"
+ "github.com/apache/arrow/go/arrow/internal/flatbuf"
+ "github.com/apache/arrow/go/arrow/memory"
+ "github.com/pkg/errors"
+)
+
+// Reader reads records from an io.Reader.
+// Reader expects a schema (plus any dictionaries) as the first messages
+// in the stream, followed by records.
+type Reader struct {
+ r *MessageReader
+ schema *arrow.Schema
+
+ refCount int64
+ rec array.Record
+ err error
+
+ types dictTypeMap
+ memo dictMemo
+
+ mem memory.Allocator
+
+ done bool
+}
+
+// NewReader returns a reader that reads records from an input stream.
+func NewReader(r io.Reader, opts ...Option) (*Reader, error) {
+ cfg := newConfig()
+ for _, opt := range opts {
+ opt(cfg)
+ }
+
+ rr := &Reader{
+ r: NewMessageReader(r),
+ types: make(dictTypeMap),
+ memo: newMemo(),
+ mem: cfg.alloc,
+ }
+
+ err := rr.readSchema()
+ if err != nil {
+ return nil, errors.Wrap(err, "arrow/ipc: could not read schema from stream")
+ }
+
+ return rr, nil
+}
+
+// Err returns the last error encountered during the iteration over the
+// underlying stream.
+func (r *Reader) Err() error { return r.err }
+
+func (r *Reader) Schema() *arrow.Schema { return r.schema }
+
+func (r *Reader) readSchema() error {
+ msg, err := r.r.Message()
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could not read message schema")
+ }
+
+ if msg.Type() != MessageSchema {
+ return errors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v)", msg.Type(), MessageSchema)
+ }
+
+ // FIXME(sbinet) refactor msg-header handling.
+ var schemaFB flatbuf.Schema
+ initFB(&schemaFB, msg.msg.Header)
+
+ r.types, err = dictTypesFromFB(&schemaFB)
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could read dictionary types from message schema")
+ }
+
+ // TODO(sbinet): in the future, we may want to reconcile IDs in the stream with
+ // those found in the schema.
+ for range r.types {
+ panic("not implemented") // FIXME(sbinet): ReadNextDictionary
+ }
+
+ r.schema, err = schemaFromFB(&schemaFB, &r.memo)
+ if err != nil {
+ return errors.Wrap(err, "arrow/ipc: could not decode schema from message schema")
+ }
+
+ return nil
+}
+
+// Retain increases the reference count by 1.
+// Retain may be called simultaneously from multiple goroutines.
+func (r *Reader) Retain() {
+ atomic.AddInt64(&r.refCount, 1)
+}
+
+// Release decreases the reference count by 1.
+// When the reference count goes to zero, the memory is freed.
+// Release may be called simultaneously from multiple goroutines.
+func (r *Reader) Release() {
+ debug.Assert(atomic.LoadInt64(&r.refCount) > 0, "too many releases")
+
+ if atomic.AddInt64(&r.refCount, -1) == 0 {
+ if r.rec != nil {
+ r.rec.Release()
+ r.rec = nil
+ }
+ if r.r != nil {
+ r.r.Release()
+ r.r = nil
+ }
+ }
+}
+
+// Next returns whether a Record could be extracted from the underlying stream.
+func (r *Reader) Next() bool {
+ if r.rec != nil {
+ r.rec.Release()
+ r.rec = nil
+ }
+
+ if r.err != nil || r.done {
+ return false
+ }
+
+ return r.next()
+}
+
+func (r *Reader) next() bool {
+ var msg *Message
+ msg, r.err = r.r.Message()
+ if r.err != nil {
+ r.done = true
+ if r.err == io.EOF {
+ r.err = nil
+ }
+ return false
+ }
+
+ if got, want := msg.Type(), MessageRecordBatch; got != want {
+ r.err = errors.Errorf("arrow/ipc: invalid message type (got=%v, want=%v", got, want)
+ return false
+ }
+
+ r.rec = newRecord(r.schema, msg.meta, bytes.NewReader(msg.body.Bytes()))
+ return true
+}
+
+// Record returns the current record that has been extracted from the
+// underlying stream.
+// It is valid until the next call to Next.
+func (r *Reader) Record() array.Record {
+ return r.rec
+}
+
+var (
+ _ array.RecordReader = (*Reader)(nil)
+)
diff --git a/go/arrow/schema.go b/go/arrow/schema.go
index a0cccc2..0083707 100644
--- a/go/arrow/schema.go
+++ b/go/arrow/schema.go
@@ -19,6 +19,7 @@ package arrow
import (
"fmt"
"sort"
+ "strings"
)
type Metadata struct {
@@ -64,6 +65,30 @@ func (md Metadata) Len() int { return len(md.keys) }
func (md Metadata) Keys() []string { return md.keys }
func (md Metadata) Values() []string { return md.values }
+func (md Metadata) String() string {
+ o := new(strings.Builder)
+ fmt.Fprintf(o, "[")
+ for i := range md.keys {
+ if i > 0 {
+ fmt.Fprintf(o, ", ")
+ }
+ fmt.Fprintf(o, "%q: %q", md.keys[i], md.values[i])
+ }
+ fmt.Fprintf(o, "]")
+ return o.String()
+}
+
+// FindKey returns the index of the key-value pair with the provided key name,
+// or -1 if such a key does not exist.
+func (md Metadata) FindKey(k string) int {
+ for i, v := range md.keys {
+ if v == k {
+ return i
+ }
+ }
+ return -1
+}
+
func (md Metadata) clone() Metadata {
if len(md.keys) == 0 {
return Metadata{}
diff --git a/go/arrow/schema_test.go b/go/arrow/schema_test.go
index 4ad36c0..0f2a029 100644
--- a/go/arrow/schema_test.go
+++ b/go/arrow/schema_test.go
@@ -91,6 +91,34 @@ func TestMetadata(t *testing.T) {
}
})
}
+
+ t.Run("find-key", func(t *testing.T) {
+ md := NewMetadata([]string{"k1", "k11"}, []string{"v1", "v11"})
+
+ if got, want := md.FindKey("k1"), 0; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := md.FindKey(""), -1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := md.FindKey("k"), -1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := md.FindKey(" "), -1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := md.FindKey("k11"), 1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+
+ if got, want := md.FindKey("k11 "), -1; got != want {
+ t.Fatalf("got=%d, want=%d", got, want)
+ }
+ })
}
func TestSchema(t *testing.T) {