You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/04/30 17:19:27 UTC
arrow git commit: ARROW-917: [GLib] Add GArrowBufferReader
Repository: arrow
Updated Branches:
refs/heads/master ed5a1d4f9 -> ce0c96221
ARROW-917: [GLib] Add GArrowBufferReader
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #617 from kou/glib-buffer-reader and squashes the following commits:
399acda [Kouhei Sutou] [GLib] Add GArrowBufferReader
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/ce0c9622
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/ce0c9622
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/ce0c9622
Branch: refs/heads/master
Commit: ce0c96221f0db74b51af5484bd39f0619b71e58f
Parents: ed5a1d4
Author: Kouhei Sutou <ko...@clear-code.com>
Authored: Sun Apr 30 13:19:22 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Apr 30 13:19:22 2017 -0400
----------------------------------------------------------------------
c_glib/arrow-glib/input-stream.cpp | 232 ++++++++++++++++++++++++--
c_glib/arrow-glib/input-stream.h | 102 +++++++++--
c_glib/arrow-glib/input-stream.hpp | 16 +-
c_glib/arrow-glib/memory-mapped-file.cpp | 17 --
c_glib/test/test-buffer-reader.rb | 26 +++
5 files changed, 341 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/ce0c9622/c_glib/arrow-glib/input-stream.cpp
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/input-stream.cpp b/c_glib/arrow-glib/input-stream.cpp
index 36bef80..56b811a 100644
--- a/c_glib/arrow-glib/input-stream.cpp
+++ b/c_glib/arrow-glib/input-stream.cpp
@@ -21,36 +21,246 @@
# include <config.h>
#endif
-#include <arrow/api.h>
+#include <arrow/io/interfaces.h>
+#include <arrow/io/memory.h>
+#include <arrow-glib/buffer.hpp>
#include <arrow-glib/error.hpp>
+#include <arrow-glib/file.hpp>
#include <arrow-glib/input-stream.hpp>
+#include <arrow-glib/random-access-file.hpp>
+#include <arrow-glib/readable.hpp>
G_BEGIN_DECLS
/**
* SECTION: input-stream
- * @title: GArrowInputStream
- * @short_description: Stream input interface
+ * @section_id: input-stream-classes
+ * @title: Input stream classes
+ * @include: arrow-glib/arrow-glib.h
*
- * #GArrowInputStream is an interface for stream input. Stream input
- * is file based and readable.
+ * #GArrowInputStream is a base class for input stream.
+ *
+ * #GArrowBufferReader is a class for buffer input stream.
*/
-G_DEFINE_INTERFACE(GArrowInputStream,
- garrow_input_stream,
- G_TYPE_OBJECT)
+typedef struct GArrowInputStreamPrivate_ {
+ std::shared_ptr<arrow::io::InputStream> input_stream;
+} GArrowInputStreamPrivate;
+
+enum {
+ PROP_0,
+ PROP_INPUT_STREAM
+};
+
+static std::shared_ptr<arrow::io::FileInterface>
+garrow_input_stream_get_raw_file_interface(GArrowFile *file)
+{
+ auto input_stream = GARROW_INPUT_STREAM(file);
+ auto arrow_input_stream =
+ garrow_input_stream_get_raw(input_stream);
+ return arrow_input_stream;
+}
+
+static void
+garrow_input_stream_file_interface_init(GArrowFileInterface *iface)
+{
+ iface->get_raw = garrow_input_stream_get_raw_file_interface;
+}
+
+static std::shared_ptr<arrow::io::Readable>
+garrow_input_stream_get_raw_readable_interface(GArrowReadable *readable)
+{
+ auto input_stream = GARROW_INPUT_STREAM(readable);
+ auto arrow_input_stream = garrow_input_stream_get_raw(input_stream);
+ return arrow_input_stream;
+}
+
+static void
+garrow_input_stream_readable_interface_init(GArrowReadableInterface *iface)
+{
+ iface->get_raw = garrow_input_stream_get_raw_readable_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowInputStream,
+ garrow_input_stream,
+ G_TYPE_OBJECT,
+ G_ADD_PRIVATE(GArrowInputStream)
+ G_IMPLEMENT_INTERFACE(GARROW_TYPE_FILE,
+ garrow_input_stream_file_interface_init)
+ G_IMPLEMENT_INTERFACE(GARROW_TYPE_READABLE,
+ garrow_input_stream_readable_interface_init));
+
+#define GARROW_INPUT_STREAM_GET_PRIVATE(obj) \
+ (G_TYPE_INSTANCE_GET_PRIVATE((obj), \
+ GARROW_TYPE_INPUT_STREAM, \
+ GArrowInputStreamPrivate))
+
+static void
+garrow_input_stream_finalize(GObject *object)
+{
+ GArrowInputStreamPrivate *priv;
+
+ priv = GARROW_INPUT_STREAM_GET_PRIVATE(object);
+
+ priv->input_stream = nullptr;
+
+ G_OBJECT_CLASS(garrow_input_stream_parent_class)->finalize(object);
+}
+
+static void
+garrow_input_stream_set_property(GObject *object,
+ guint prop_id,
+ const GValue *value,
+ GParamSpec *pspec)
+{
+ GArrowInputStreamPrivate *priv;
+
+ priv = GARROW_INPUT_STREAM_GET_PRIVATE(object);
+
+ switch (prop_id) {
+ case PROP_INPUT_STREAM:
+ priv->input_stream =
+ *static_cast<std::shared_ptr<arrow::io::InputStream> *>(g_value_get_pointer(value));
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+garrow_input_stream_get_property(GObject *object,
+ guint prop_id,
+ GValue *value,
+ GParamSpec *pspec)
+{
+ switch (prop_id) {
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
+ break;
+ }
+}
+
+static void
+garrow_input_stream_init(GArrowInputStream *object)
+{
+}
+
+static void
+garrow_input_stream_class_init(GArrowInputStreamClass *klass)
+{
+ GObjectClass *gobject_class;
+ GParamSpec *spec;
+
+ gobject_class = G_OBJECT_CLASS(klass);
+
+ gobject_class->finalize = garrow_input_stream_finalize;
+ gobject_class->set_property = garrow_input_stream_set_property;
+ gobject_class->get_property = garrow_input_stream_get_property;
+
+ spec = g_param_spec_pointer("input-stream",
+ "Input stream",
+ "The raw std::shared<arrow::io::InputStream> *",
+ static_cast<GParamFlags>(G_PARAM_WRITABLE |
+ G_PARAM_CONSTRUCT_ONLY));
+ g_object_class_install_property(gobject_class, PROP_INPUT_STREAM, spec);
+}
+
+
+static std::shared_ptr<arrow::io::RandomAccessFile>
+garrow_buffer_reader_get_raw_random_access_file_interface(GArrowRandomAccessFile *random_access_file)
+{
+ auto input_stream = GARROW_INPUT_STREAM(random_access_file);
+ auto arrow_input_stream = garrow_input_stream_get_raw(input_stream);
+ auto arrow_buffer_reader =
+ std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
+ return arrow_buffer_reader;
+}
static void
-garrow_input_stream_default_init (GArrowInputStreamInterface *iface)
+garrow_buffer_reader_random_access_file_interface_init(GArrowRandomAccessFileInterface *iface)
+{
+ iface->get_raw = garrow_buffer_reader_get_raw_random_access_file_interface;
+}
+
+G_DEFINE_TYPE_WITH_CODE(GArrowBufferReader, \
+ garrow_buffer_reader, \
+ GARROW_TYPE_INPUT_STREAM,
+ G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
+ garrow_buffer_reader_random_access_file_interface_init));
+
+static void
+garrow_buffer_reader_init(GArrowBufferReader *object)
+{
+}
+
+static void
+garrow_buffer_reader_class_init(GArrowBufferReaderClass *klass)
+{
+}
+
+/**
+ * garrow_buffer_reader_new:
+ * @buffer: The buffer to be read.
+ *
+ * Returns: A newly created #GArrowBufferReader.
+ */
+GArrowBufferReader *
+garrow_buffer_reader_new(GArrowBuffer *buffer)
+{
+ auto arrow_buffer = garrow_buffer_get_raw(buffer);
+ auto arrow_buffer_reader =
+ std::make_shared<arrow::io::BufferReader>(arrow_buffer);
+ return garrow_buffer_reader_new_raw(&arrow_buffer_reader);
+}
+
+/**
+ * garrow_buffer_reader_get_buffer:
+ * @buffer_reader: A #GArrowBufferReader.
+ *
+ * Returns: (transfer full): The data of the array as #GArrowBuffer.
+ */
+GArrowBuffer *
+garrow_buffer_reader_get_buffer(GArrowBufferReader *buffer_reader)
{
+ auto arrow_input_stream =
+ garrow_input_stream_get_raw(GARROW_INPUT_STREAM(buffer_reader));
+ auto arrow_buffer_reader =
+ std::static_pointer_cast<arrow::io::BufferReader>(arrow_input_stream);
+ auto arrow_buffer = arrow_buffer_reader->buffer();
+ return garrow_buffer_new_raw(&arrow_buffer);
}
+
G_END_DECLS
+GArrowInputStream *
+garrow_input_stream_new_raw(std::shared_ptr<arrow::io::InputStream> *arrow_input_stream)
+{
+ auto input_stream =
+ GARROW_INPUT_STREAM(g_object_new(GARROW_TYPE_INPUT_STREAM,
+ "input-stream", arrow_input_stream,
+ NULL));
+ return input_stream;
+}
+
std::shared_ptr<arrow::io::InputStream>
garrow_input_stream_get_raw(GArrowInputStream *input_stream)
{
- auto *iface = GARROW_INPUT_STREAM_GET_IFACE(input_stream);
- return iface->get_raw(input_stream);
+ GArrowInputStreamPrivate *priv;
+
+ priv = GARROW_INPUT_STREAM_GET_PRIVATE(input_stream);
+ return priv->input_stream;
+}
+
+
+GArrowBufferReader *
+garrow_buffer_reader_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader)
+{
+ auto buffer_reader =
+ GARROW_BUFFER_READER(g_object_new(GARROW_TYPE_BUFFER_READER,
+ "input-stream", arrow_buffer_reader,
+ NULL));
+ return buffer_reader;
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/ce0c9622/c_glib/arrow-glib/input-stream.h
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/input-stream.h b/c_glib/arrow-glib/input-stream.h
index 4b331b9..caa11b5 100644
--- a/c_glib/arrow-glib/input-stream.h
+++ b/c_glib/arrow-glib/input-stream.h
@@ -19,27 +19,105 @@
#pragma once
-#include <glib-object.h>
+#include <arrow-glib/buffer.h>
G_BEGIN_DECLS
-#define GARROW_TYPE_INPUT_STREAM \
+#define GARROW_TYPE_INPUT_STREAM \
(garrow_input_stream_get_type())
-#define GARROW_INPUT_STREAM(obj) \
- (G_TYPE_CHECK_INSTANCE_CAST((obj), \
- GARROW_TYPE_INPUT_STREAM, \
+#define GARROW_INPUT_STREAM(obj) \
+ (G_TYPE_CHECK_INSTANCE_CAST((obj), \
+ GARROW_TYPE_INPUT_STREAM, \
GArrowInputStream))
-#define GARROW_IS_INPUT_STREAM(obj) \
+#define GARROW_INPUT_STREAM_CLASS(klass) \
+ (G_TYPE_CHECK_CLASS_CAST((klass), \
+ GARROW_TYPE_INPUT_STREAM, \
+ GArrowInputStreamClass))
+#define GARROW_IS_INPUT_STREAM(obj) \
(G_TYPE_CHECK_INSTANCE_TYPE((obj), \
GARROW_TYPE_INPUT_STREAM))
-#define GARROW_INPUT_STREAM_GET_IFACE(obj) \
- (G_TYPE_INSTANCE_GET_INTERFACE((obj), \
- GARROW_TYPE_INPUT_STREAM, \
- GArrowInputStreamInterface))
+#define GARROW_IS_INPUT_STREAM_CLASS(klass) \
+ (G_TYPE_CHECK_CLASS_TYPE((klass), \
+ GARROW_TYPE_INPUT_STREAM))
+#define GARROW_INPUT_STREAM_GET_CLASS(obj) \
+ (G_TYPE_INSTANCE_GET_CLASS((obj), \
+ GARROW_TYPE_INPUT_STREAM, \
+ GArrowInputStreamClass))
+
+typedef struct _GArrowInputStream GArrowInputStream;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowInputStreamClass GArrowInputStreamClass;
+#endif
+
+/**
+ * GArrowInputStream:
+ *
+ * It wraps `arrow::io::InputStream`.
+ */
+struct _GArrowInputStream
+{
+ /*< private >*/
+ GObject parent_instance;
+};
-typedef struct _GArrowInputStream GArrowInputStream;
-typedef struct _GArrowInputStreamInterface GArrowInputStreamInterface;
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowInputStreamClass
+{
+ GObjectClass parent_class;
+};
+#endif
GType garrow_input_stream_get_type(void) G_GNUC_CONST;
+
+#define GARROW_TYPE_BUFFER_READER \
+ (garrow_buffer_reader_get_type())
+#define GARROW_BUFFER_READER(obj) \
+ (G_TYPE_CHECK_INSTANCE_CAST((obj), \
+ GARROW_TYPE_BUFFER_READER, \
+ GArrowBufferReader))
+#define GARROW_BUFFER_READER_CLASS(klass) \
+ (G_TYPE_CHECK_CLASS_CAST((klass), \
+ GARROW_TYPE_BUFFER_READER, \
+ GArrowBufferReaderClass))
+#define GARROW_IS_BUFFER_READER(obj) \
+ (G_TYPE_CHECK_INSTANCE_TYPE((obj), \
+ GARROW_TYPE_BUFFER_READER))
+#define GARROW_IS_BUFFER_READER_CLASS(klass) \
+ (G_TYPE_CHECK_CLASS_TYPE((klass), \
+ GARROW_TYPE_BUFFER_READER))
+#define GARROW_BUFFER_READER_GET_CLASS(obj) \
+ (G_TYPE_INSTANCE_GET_CLASS((obj), \
+ GARROW_TYPE_BUFFER_READER, \
+ GArrowBufferReaderClass))
+
+typedef struct _GArrowBufferReader GArrowBufferReader;
+#ifndef __GTK_DOC_IGNORE__
+typedef struct _GArrowBufferReaderClass GArrowBufferReaderClass;
+#endif
+
+/**
+ * GArrowBufferReader:
+ *
+ * It wraps `arrow::io::BufferReader`.
+ */
+struct _GArrowBufferReader
+{
+ /*< private >*/
+ GArrowInputStream parent_instance;
+};
+
+#ifndef __GTK_DOC_IGNORE__
+struct _GArrowBufferReaderClass
+{
+ GArrowInputStreamClass parent_class;
+};
+#endif
+
+GType garrow_buffer_reader_get_type(void) G_GNUC_CONST;
+
+GArrowBufferReader *garrow_buffer_reader_new(GArrowBuffer *buffer);
+
+GArrowBuffer *garrow_buffer_reader_get_buffer(GArrowBufferReader *buffer_reader);
+
G_END_DECLS
http://git-wip-us.apache.org/repos/asf/arrow/blob/ce0c9622/c_glib/arrow-glib/input-stream.hpp
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/input-stream.hpp b/c_glib/arrow-glib/input-stream.hpp
index 7958df1..008f5f2 100644
--- a/c_glib/arrow-glib/input-stream.hpp
+++ b/c_glib/arrow-glib/input-stream.hpp
@@ -20,19 +20,11 @@
#pragma once
#include <arrow/io/interfaces.h>
+#include <arrow/io/memory.h>
#include <arrow-glib/input-stream.h>
-/**
- * GArrowInputStreamInterface:
- *
- * It wraps `arrow::io::InputStream`.
- */
-struct _GArrowInputStreamInterface
-{
- GTypeInterface parent_iface;
-
- std::shared_ptr<arrow::io::InputStream> (*get_raw)(GArrowInputStream *file);
-};
-
+GArrowInputStream *garrow_input_stream_new_raw(std::shared_ptr<arrow::io::InputStream> *arrow_input_stream);
std::shared_ptr<arrow::io::InputStream> garrow_input_stream_get_raw(GArrowInputStream *input_stream);
+
+GArrowBufferReader *garrow_buffer_reader_new_raw(std::shared_ptr<arrow::io::BufferReader> *arrow_buffer_reader);
http://git-wip-us.apache.org/repos/asf/arrow/blob/ce0c9622/c_glib/arrow-glib/memory-mapped-file.cpp
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/memory-mapped-file.cpp b/c_glib/arrow-glib/memory-mapped-file.cpp
index f9cbf07..71a9a6d 100644
--- a/c_glib/arrow-glib/memory-mapped-file.cpp
+++ b/c_glib/arrow-glib/memory-mapped-file.cpp
@@ -82,21 +82,6 @@ garrow_readable_interface_init(GArrowReadableInterface *iface)
iface->get_raw = garrow_memory_mapped_file_get_raw_readable_interface;
}
-static std::shared_ptr<arrow::io::InputStream>
-garrow_memory_mapped_file_get_raw_input_stream_interface(GArrowInputStream *input_stream)
-{
- auto memory_mapped_file = GARROW_MEMORY_MAPPED_FILE(input_stream);
- auto arrow_memory_mapped_file =
- garrow_memory_mapped_file_get_raw(memory_mapped_file);
- return arrow_memory_mapped_file;
-}
-
-static void
-garrow_input_stream_interface_init(GArrowInputStreamInterface *iface)
-{
- iface->get_raw = garrow_memory_mapped_file_get_raw_input_stream_interface;
-}
-
static std::shared_ptr<arrow::io::RandomAccessFile>
garrow_memory_mapped_file_get_raw_random_access_file_interface(GArrowRandomAccessFile *file)
{
@@ -150,8 +135,6 @@ G_DEFINE_TYPE_WITH_CODE(GArrowMemoryMappedFile,
garrow_file_interface_init)
G_IMPLEMENT_INTERFACE(GARROW_TYPE_READABLE,
garrow_readable_interface_init)
- G_IMPLEMENT_INTERFACE(GARROW_TYPE_INPUT_STREAM,
- garrow_input_stream_interface_init)
G_IMPLEMENT_INTERFACE(GARROW_TYPE_RANDOM_ACCESS_FILE,
garrow_random_access_file_interface_init)
G_IMPLEMENT_INTERFACE(GARROW_TYPE_WRITEABLE,
http://git-wip-us.apache.org/repos/asf/arrow/blob/ce0c9622/c_glib/test/test-buffer-reader.rb
----------------------------------------------------------------------
diff --git a/c_glib/test/test-buffer-reader.rb b/c_glib/test/test-buffer-reader.rb
new file mode 100644
index 0000000..b3517b2
--- /dev/null
+++ b/c_glib/test/test-buffer-reader.rb
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class TestBufferReader < Test::Unit::TestCase
+ def test_read
+ buffer = Arrow::Buffer.new("Hello World")
+ buffer_reader = Arrow::BufferReader.new(buffer)
+ read_buffer = " " * 5
+ _success, n_read_bytes = buffer_reader.read(read_buffer)
+ assert_equal("Hello", read_buffer.byteslice(0, n_read_bytes))
+ end
+end