You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/04/30 17:35:35 UTC
arrow git commit: ARROW-920: [GLib] Add Lua examples
Repository: arrow
Updated Branches:
refs/heads/master b4886da0f -> 00994b820
ARROW-920: [GLib] Add Lua examples
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #620 from kou/glib-lua-examples and squashes the following commits:
491c0e4 [Kouhei Sutou] [GLib] Add Lua examples
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/00994b82
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/00994b82
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/00994b82
Branch: refs/heads/master
Commit: 00994b82015365fec8474605bf09bd11637859af
Parents: b4886da
Author: Kouhei Sutou <ko...@clear-code.com>
Authored: Sun Apr 30 13:35:29 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Apr 30 13:35:29 2017 -0400
----------------------------------------------------------------------
c_glib/README.md | 2 +
c_glib/configure.ac | 1 +
c_glib/example/Makefile.am | 6 ++-
c_glib/example/README.md | 5 ++-
c_glib/example/lua/Makefile.am | 24 +++++++++++
c_glib/example/lua/README.md | 45 +++++++++++++++++++
c_glib/example/lua/read-batch.lua | 44 +++++++++++++++++++
c_glib/example/lua/read-stream.lua | 51 ++++++++++++++++++++++
c_glib/example/lua/write-batch.lua | 74 ++++++++++++++++++++++++++++++++
c_glib/example/lua/write-stream.lua | 74 ++++++++++++++++++++++++++++++++
10 files changed, 323 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/README.md
----------------------------------------------------------------------
diff --git a/c_glib/README.md b/c_glib/README.md
index b253d32..6eadb79 100644
--- a/c_glib/README.md
+++ b/c_glib/README.md
@@ -194,10 +194,12 @@ You can use Arrow GLib with non C languages with GObject Introspection
based bindings. Here are languages that support GObject Introspection:
* Ruby: [red-arrow gem](https://rubygems.org/gems/red-arrow) should be used.
+ * Examples: https://github.com/red-data-tools/red-arrow/tree/master/example
* Python: [PyGObject](https://wiki.gnome.org/Projects/PyGObject) should be used. (Note that you should use PyArrow than Arrow GLib.)
* Lua: [LGI](https://github.com/pavouk/lgi) should be used.
+ * Examples: `example/lua/` directory.
* Go: [Go-gir-generator](https://github.com/linuxdeepin/go-gir-generator) should be used.
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/configure.ac
----------------------------------------------------------------------
diff --git a/c_glib/configure.ac b/c_glib/configure.ac
index f367192..e010d96 100644
--- a/c_glib/configure.ac
+++ b/c_glib/configure.ac
@@ -97,6 +97,7 @@ AC_CONFIG_FILES([
doc/Makefile
doc/reference/Makefile
example/Makefile
+ example/lua/Makefile
])
AC_OUTPUT
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/Makefile.am
----------------------------------------------------------------------
diff --git a/c_glib/example/Makefile.am b/c_glib/example/Makefile.am
index 8bf3c15..66d2cdd 100644
--- a/c_glib/example/Makefile.am
+++ b/c_glib/example/Makefile.am
@@ -15,6 +15,9 @@
# specific language governing permissions and limitations
# under the License.
+SUBDIRS = \
+ lua
+
AM_CPPFLAGS = \
-I$(top_builddir) \
-I$(top_srcdir)
@@ -41,7 +44,8 @@ read_batch_SOURCES = \
read_stream_SOURCES = \
read-stream.c
-example_DATA = \
+dist_example_DATA = \
+ README.md \
$(build_SOURCES) \
$(read_batch_SOURCES) \
$(read_stream_SOURCES)
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/README.md
----------------------------------------------------------------------
diff --git a/c_glib/example/README.md b/c_glib/example/README.md
index b1ba259..99730d5 100644
--- a/c_glib/example/README.md
+++ b/c_glib/example/README.md
@@ -16,7 +16,9 @@
There are example codes in this directory.
-C example codes exist in this directory.
+C example codes exist in this directory. Language bindings example
+codes exists in sub directories. For example, Lua example codes exists
+in `lua/` sub directory.
## C example codes
@@ -39,4 +41,3 @@ Here are example codes in this directory:
* `read-stream.c`: It shows how to read Arrow array from file in
stream mode.
-
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/Makefile.am
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/Makefile.am b/c_glib/example/lua/Makefile.am
new file mode 100644
index 0000000..9019d24
--- /dev/null
+++ b/c_glib/example/lua/Makefile.am
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+lua_exampledir = $(exampledir)/lua
+dist_lua_example_DATA = \
+ README.md \
+ read-batch.lua \
+ read-stream.lua \
+ write-batch.lua \
+ write-stream.lua
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/README.md
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md
new file mode 100644
index 0000000..d127573
--- /dev/null
+++ b/c_glib/example/lua/README.md
@@ -0,0 +1,45 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Arrow Lua example
+
+There are Lua example codes in this directory.
+
+## How to run
+
+All example codes use [LGI](https://github.com/pavouk/lgi) to use
+Arrow GLib based bindings.
+
+Here are command lines to install LGI on Debian GNU/Linux and Ubuntu:
+
+```text
+% sudo apt install -y luarocks
+% sudo luarocks install lgi
+```
+
+## Lua example codes
+
+Here are example codes in this directory:
+
+ * `write-batch.lua`: It shows how to write Arrow array to file in
+ batch mode.
+
+ * `read-batch.lua`: It shows how to read Arrow array from file in
+ batch mode.
+
+ * `write-stream.lua`: It shows how to write Arrow array to file in
+ stream mode.
+
+ * `read-stream.lua`: It shows how to read Arrow array from file in
+ stream mode.
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/read-batch.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
new file mode 100644
index 0000000..b28d346
--- /dev/null
+++ b/c_glib/example/lua/read-batch.lua
@@ -0,0 +1,44 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local input_path = arg[1] or "/tmp/batch.arrow";
+
+local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ)
+local reader = Arrow.FileReader.open(input)
+
+for i = 0, reader:get_n_record_batches() - 1 do
+ local record_batch = reader:get_record_batch(i)
+ print(string.rep("=", 40))
+ print("record-batch["..i.."]:")
+ for j = 0, record_batch:get_n_columns() - 1 do
+ local column = record_batch:get_column(j)
+ local column_name = record_batch:get_column_name(j)
+ io.write(" "..column_name..": [")
+ for k = 0, record_batch:get_n_rows() - 1 do
+ if k > 0 then
+ io.write(", ")
+ end
+ io.write(column:get_value(k))
+ end
+ print("]")
+ end
+end
+
+input:close()
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/read-stream.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
new file mode 100644
index 0000000..3b08206
--- /dev/null
+++ b/c_glib/example/lua/read-stream.lua
@@ -0,0 +1,51 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local input_path = arg[1] or "/tmp/stream.arrow";
+
+local input = Arrow.MemoryMappedFile.open(input_path, Arrow.FileMode.READ)
+local reader = Arrow.StreamReader.open(input)
+
+local i = 0
+while true do
+ local record_batch = reader:get_next_record_batch(i)
+ if not record_batch then
+ break
+ end
+
+ print(string.rep("=", 40))
+ print("record-batch["..i.."]:")
+ for j = 0, record_batch:get_n_columns() - 1 do
+ local column = record_batch:get_column(j)
+ local column_name = record_batch:get_column_name(j)
+ io.write(" "..column_name..": [")
+ for k = 0, record_batch:get_n_rows() - 1 do
+ if k > 0 then
+ io.write(", ")
+ end
+ io.write(column:get_value(k))
+ end
+ print("]")
+ end
+
+ i = i + 1
+end
+
+input:close()
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/write-batch.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/write-batch.lua b/c_glib/example/lua/write-batch.lua
new file mode 100644
index 0000000..3a22cd5
--- /dev/null
+++ b/c_glib/example/lua/write-batch.lua
@@ -0,0 +1,74 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local output_path = arg[1] or "/tmp/batch.arrow";
+
+local fields = {
+ Arrow.Field.new("uint8", Arrow.UInt8DataType.new()),
+ Arrow.Field.new("uint16", Arrow.UInt16DataType.new()),
+ Arrow.Field.new("uint32", Arrow.UInt32DataType.new()),
+ Arrow.Field.new("uint64", Arrow.UInt64DataType.new()),
+ Arrow.Field.new("int8", Arrow.Int8DataType.new()),
+ Arrow.Field.new("int16", Arrow.Int16DataType.new()),
+ Arrow.Field.new("int32", Arrow.Int32DataType.new()),
+ Arrow.Field.new("int64", Arrow.Int64DataType.new()),
+ Arrow.Field.new("float", Arrow.FloatDataType.new()),
+ Arrow.Field.new("double", Arrow.DoubleDataType.new()),
+}
+local schema = Arrow.Schema.new(fields)
+
+local output = Arrow.FileOutputStream.open(output_path, false)
+local writer = Arrow.FileWriter.open(output, schema)
+
+function build_array(builder, values)
+ for _, value in pairs(values) do
+ builder:append(value)
+ end
+ return builder:finish()
+end
+
+local uints = {1, 2, 4, 8}
+local ints = {1, -2, 4, -8}
+local floats = {1.1, -2.2, 4.4, -8.8}
+local columns = {
+ build_array(Arrow.UInt8ArrayBuilder.new(), uints),
+ build_array(Arrow.UInt16ArrayBuilder.new(), uints),
+ build_array(Arrow.UInt32ArrayBuilder.new(), uints),
+ build_array(Arrow.UInt64ArrayBuilder.new(), uints),
+ build_array(Arrow.Int8ArrayBuilder.new(), ints),
+ build_array(Arrow.Int16ArrayBuilder.new(), ints),
+ build_array(Arrow.Int32ArrayBuilder.new(), ints),
+ build_array(Arrow.Int64ArrayBuilder.new(), ints),
+ build_array(Arrow.FloatArrayBuilder.new(), floats),
+ build_array(Arrow.DoubleArrayBuilder.new(), floats),
+}
+
+local record_batch = Arrow.RecordBatch.new(schema, 4, columns)
+writer:write_record_batch(record_batch)
+
+local sliced_columns = {}
+for i, column in pairs(columns) do
+ sliced_columns[i] = column:slice(1, 3)
+end
+record_batch = Arrow.RecordBatch.new(schema, 3, sliced_columns)
+writer:write_record_batch(record_batch)
+
+writer:close()
+output:close()
http://git-wip-us.apache.org/repos/asf/arrow/blob/00994b82/c_glib/example/lua/write-stream.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/write-stream.lua b/c_glib/example/lua/write-stream.lua
new file mode 100644
index 0000000..37c6bb5
--- /dev/null
+++ b/c_glib/example/lua/write-stream.lua
@@ -0,0 +1,74 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local output_path = arg[1] or "/tmp/stream.arrow";
+
+local fields = {
+ Arrow.Field.new("uint8", Arrow.UInt8DataType.new()),
+ Arrow.Field.new("uint16", Arrow.UInt16DataType.new()),
+ Arrow.Field.new("uint32", Arrow.UInt32DataType.new()),
+ Arrow.Field.new("uint64", Arrow.UInt64DataType.new()),
+ Arrow.Field.new("int8", Arrow.Int8DataType.new()),
+ Arrow.Field.new("int16", Arrow.Int16DataType.new()),
+ Arrow.Field.new("int32", Arrow.Int32DataType.new()),
+ Arrow.Field.new("int64", Arrow.Int64DataType.new()),
+ Arrow.Field.new("float", Arrow.FloatDataType.new()),
+ Arrow.Field.new("double", Arrow.DoubleDataType.new()),
+}
+local schema = Arrow.Schema.new(fields)
+
+local output = Arrow.FileOutputStream.open(output_path, false)
+local writer = Arrow.StreamWriter.open(output, schema)
+
+function build_array(builder, values)
+ for _, value in pairs(values) do
+ builder:append(value)
+ end
+ return builder:finish()
+end
+
+local uints = {1, 2, 4, 8}
+local ints = {1, -2, 4, -8}
+local floats = {1.1, -2.2, 4.4, -8.8}
+local columns = {
+ build_array(Arrow.UInt8ArrayBuilder.new(), uints),
+ build_array(Arrow.UInt16ArrayBuilder.new(), uints),
+ build_array(Arrow.UInt32ArrayBuilder.new(), uints),
+ build_array(Arrow.UInt64ArrayBuilder.new(), uints),
+ build_array(Arrow.Int8ArrayBuilder.new(), ints),
+ build_array(Arrow.Int16ArrayBuilder.new(), ints),
+ build_array(Arrow.Int32ArrayBuilder.new(), ints),
+ build_array(Arrow.Int64ArrayBuilder.new(), ints),
+ build_array(Arrow.FloatArrayBuilder.new(), floats),
+ build_array(Arrow.DoubleArrayBuilder.new(), floats),
+}
+
+local record_batch = Arrow.RecordBatch.new(schema, 4, columns)
+writer:write_record_batch(record_batch)
+
+local sliced_columns = {}
+for i, column in pairs(columns) do
+ sliced_columns[i] = column:slice(1, 3)
+end
+record_batch = Arrow.RecordBatch.new(schema, 3, sliced_columns)
+writer:write_record_batch(record_batch)
+
+writer:close()
+output:close()