You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/05 14:38:38 UTC
arrow git commit: ARROW-945: [GLib] Add a Lua example to show Torch
integration
Repository: arrow
Updated Branches:
refs/heads/master 80b72d43e -> bcf073c3a
ARROW-945: [GLib] Add a Lua example to show Torch integration
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #637 from kou/glib-lua-to-torch-tensor and squashes the following commits:
4aba395 [Kouhei Sutou] [GLib] Add a Lua example to show Torch integration
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/bcf073c3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/bcf073c3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/bcf073c3
Branch: refs/heads/master
Commit: bcf073c3aeca872e41f86cee14d2c43598ce3149
Parents: 80b72d4
Author: Kouhei Sutou <ko...@clear-code.com>
Authored: Fri May 5 10:38:33 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Fri May 5 10:38:33 2017 -0400
----------------------------------------------------------------------
c_glib/example/lua/Makefile.am | 1 +
c_glib/example/lua/README.md | 5 +
c_glib/example/lua/read-stream.lua | 2 +-
c_glib/example/lua/stream-to-torch-tensor.lua | 101 +++++++++++++++++++++
4 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/Makefile.am
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/Makefile.am b/c_glib/example/lua/Makefile.am
index 9019d24..86bdbed 100644
--- a/c_glib/example/lua/Makefile.am
+++ b/c_glib/example/lua/Makefile.am
@@ -20,5 +20,6 @@ dist_lua_example_DATA = \
README.md \
read-batch.lua \
read-stream.lua \
+ stream-to-torch-tensor.lua \
write-batch.lua \
write-stream.lua
http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/README.md
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/README.md b/c_glib/example/lua/README.md
index d127573..6145bc7 100644
--- a/c_glib/example/lua/README.md
+++ b/c_glib/example/lua/README.md
@@ -43,3 +43,8 @@ Here are example codes in this directory:
* `read-stream.lua`: It shows how to read Arrow array from file in
stream mode.
+
+ * `stream-to-torch-tensor.lua`: It shows how to read Arrow array
+ from file in stream mode and convert it to
+ [Torch](http://torch.ch/)'s
+ [`Tensor` object](http://torch7.readthedocs.io/en/rtd/tensor/index.html).
http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/read-stream.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index e744bed..987d463 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -25,7 +25,7 @@ local reader = Arrow.StreamReader.open(input)
local i = 0
while true do
- local record_batch = reader:get_next_record_batch(i)
+ local record_batch = reader:get_next_record_batch()
if not record_batch then
break
end
http://git-wip-us.apache.org/repos/asf/arrow/blob/bcf073c3/c_glib/example/lua/stream-to-torch-tensor.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/stream-to-torch-tensor.lua b/c_glib/example/lua/stream-to-torch-tensor.lua
new file mode 100644
index 0000000..237d759
--- /dev/null
+++ b/c_glib/example/lua/stream-to-torch-tensor.lua
@@ -0,0 +1,101 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+local lgi = require 'lgi'
+local Arrow = lgi.Arrow
+
+local torch = require 'torch'
+
+Arrow.Array.torch_types = function(self)
+ return nil
+end
+
+Arrow.Array.to_torch = function(self)
+ local types = self:torch_types()
+ if not types then
+ return nil
+ end
+
+ local storage_type = types[1]
+ local tensor_type = types[2]
+
+ local size = self:get_length()
+ local storage = storage_type(size)
+ if not storage then
+ return nil
+ end
+
+ for i = 1, size do
+ storage[i] = self:get_value(i - 1)
+ end
+ return tensor_type(storage)
+end
+
+Arrow.UInt8Array.torch_types = function(self)
+ return {torch.ByteStorage, torch.ByteTensor}
+end
+
+Arrow.Int8Array.torch_types = function(self)
+ return {torch.CharStorage, torch.CharTensor}
+end
+
+Arrow.Int16Array.torch_types = function(self)
+ return {torch.ShortStorage, torch.ShortTensor}
+end
+
+Arrow.Int32Array.torch_types = function(self)
+ return {torch.IntStorage, torch.IntTensor}
+end
+
+Arrow.Int64Array.torch_types = function(self)
+ return {torch.LongStorage, torch.LongTensor}
+end
+
+Arrow.FloatArray.torch_types = function(self)
+ return {torch.FloatStorage, torch.FloatTensor}
+end
+
+Arrow.DoubleArray.torch_types = function(self)
+ return {torch.DoubleStorage, torch.DoubleTensor}
+end
+
+
+local input_path = arg[1] or "/tmp/stream.arrow";
+
+local input = Arrow.MemoryMappedInputStream.new(input_path)
+local reader = Arrow.StreamReader.open(input)
+
+local i = 0
+while true do
+ local record_batch = reader:get_next_record_batch()
+ if not record_batch then
+ break
+ end
+
+ print(string.rep("=", 40))
+ print("record-batch["..i.."]:")
+ for j = 0, record_batch:get_n_columns() - 1 do
+ local column = record_batch:get_column(j)
+ local column_name = record_batch:get_column_name(j)
+ print(" "..column_name..":")
+ print(column:to_torch())
+ end
+
+ i = i + 1
+end
+
+input:close()