You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sh...@apache.org on 2018/12/30 00:56:19 UTC

[arrow] branch master updated: ARROW-4132: [GLib] Add more GArrowTable constructors

This is an automated email from the ASF dual-hosted git repository.

shiro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 7074889  ARROW-4132: [GLib] Add more GArrowTable constructors
7074889 is described below

commit 7074889602a2279cfa2440697040a946628f5b56
Author: Kouhei Sutou <ko...@clear-code.com>
AuthorDate: Sun Dec 30 09:56:05 2018 +0900

    ARROW-4132: [GLib] Add more GArrowTable constructors
    
    Author: Kouhei Sutou <ko...@clear-code.com>
    
    Closes #3285 from kou/glib-table-new and squashes the following commits:
    
    8bab8046 <Kouhei Sutou>  Add more GArrowTable constructors
---
 c_glib/arrow-glib/composite-array.h |   2 +
 c_glib/arrow-glib/orc-file-reader.h |   4 +-
 c_glib/arrow-glib/table.cpp         | 204 +++++++++++++++++++++++++++++++++++-
 c_glib/arrow-glib/table.h           |  33 +++++-
 c_glib/arrow-glib/version.h.in      |  23 ++++
 c_glib/test/test-table.rb           |  61 +++++++++--
 6 files changed, 310 insertions(+), 17 deletions(-)

diff --git a/c_glib/arrow-glib/composite-array.h b/c_glib/arrow-glib/composite-array.h
index c634dbf..10432e2 100644
--- a/c_glib/arrow-glib/composite-array.h
+++ b/c_glib/arrow-glib/composite-array.h
@@ -130,8 +130,10 @@ GArrowStructArray *garrow_struct_array_new(GArrowDataType *data_type,
 GArrowArray *garrow_struct_array_get_field(GArrowStructArray *array,
                                            gint i);
 
+#ifndef GARROW_DISABLE_DEPRECATED
 GARROW_DEPRECATED_IN_0_10_FOR(garrow_struct_array_flatten)
 GList *garrow_struct_array_get_fields(GArrowStructArray *array);
+#endif
 
 GARROW_AVAILABLE_IN_0_10
 GList *garrow_struct_array_flatten(GArrowStructArray *array, GError **error);
diff --git a/c_glib/arrow-glib/orc-file-reader.h b/c_glib/arrow-glib/orc-file-reader.h
index 9b2dbad..97cf1ef 100644
--- a/c_glib/arrow-glib/orc-file-reader.h
+++ b/c_glib/arrow-glib/orc-file-reader.h
@@ -39,7 +39,7 @@ garrow_orc_file_reader_new(GArrowSeekableInputStream *file,
                            GError **error);
 
 #ifndef GARROW_DISABLE_DEPRECATED
-G_GNUC_DEPRECATED_FOR(garrow_orc_file_reader_set_field_indices)
+GARROW_DEPRECATED_IN_0_12_FOR(garrow_orc_file_reader_set_field_indices)
 void
 garrow_orc_file_reader_set_field_indexes(GArrowORCFileReader *reader,
                                          const gint *field_indexes,
@@ -50,7 +50,7 @@ garrow_orc_file_reader_set_field_indices(GArrowORCFileReader *reader,
                                          const gint *field_indices,
                                          guint n_field_indices);
 #ifndef GARROW_DISABLE_DEPRECATED
-G_GNUC_DEPRECATED_FOR(garrow_orc_file_reader_get_field_indices)
+GARROW_DEPRECATED_IN_0_12_FOR(garrow_orc_file_reader_get_field_indices)
 const gint *
 garrow_orc_file_reader_get_field_indexes(GArrowORCFileReader *reader,
                                          guint *n_field_indexes);
diff --git a/c_glib/arrow-glib/table.cpp b/c_glib/arrow-glib/table.cpp
index f9e1b95..b889eb2 100644
--- a/c_glib/arrow-glib/table.cpp
+++ b/c_glib/arrow-glib/table.cpp
@@ -21,8 +21,10 @@
 #  include <config.h>
 #endif
 
+#include <arrow-glib/array.hpp>
 #include <arrow-glib/column.hpp>
 #include <arrow-glib/error.hpp>
+#include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
 
@@ -133,23 +135,219 @@ garrow_table_class_init(GArrowTableClass *klass)
  * @columns: (element-type GArrowColumn): The columns of the table.
  *
  * Returns: A newly created #GArrowTable.
+ *
+ * Deprecated: 0.12.0: Use garrow_table_new_values() instead.
  */
 GArrowTable *
 garrow_table_new(GArrowSchema *schema,
                  GList *columns)
 {
+  auto arrow_schema = garrow_schema_get_raw(schema);
   std::vector<std::shared_ptr<arrow::Column>> arrow_columns;
   for (GList *node = columns; node; node = node->next) {
-    GArrowColumn *column = GARROW_COLUMN(node->data);
+    auto column = GARROW_COLUMN(node->data);
     arrow_columns.push_back(garrow_column_get_raw(column));
   }
 
-  auto arrow_table =
-    arrow::Table::Make(garrow_schema_get_raw(schema), arrow_columns);
+  auto arrow_table = arrow::Table::Make(arrow_schema, arrow_columns);
   return garrow_table_new_raw(&arrow_table);
 }
 
 /**
+ * garrow_table_new_values: (skip)
+ * @schema: The schema of the table.
+ * @values: The values of the table. All values must be instance of the
+ *   same class. Available classes are #GArrowColumn, #GArrowArray and
+ *   #GArrowRecordBatch.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowTable or %NULL on error.
+ *
+ * Since: 0.12.0
+ */
+GArrowTable *
+garrow_table_new_values(GArrowSchema *schema,
+                        GList *values,
+                        GError **error)
+{
+  const auto context = "[table][new][values]";
+  auto arrow_schema = garrow_schema_get_raw(schema);
+  std::vector<std::shared_ptr<arrow::Column>> arrow_columns;
+  std::vector<std::shared_ptr<arrow::Array>> arrow_arrays;
+  std::vector<std::shared_ptr<arrow::RecordBatch>> arrow_record_batches;
+  for (GList *node = values; node; node = node->next) {
+    if (GARROW_IS_COLUMN(node->data)) {
+      auto column = GARROW_COLUMN(node->data);
+      arrow_columns.push_back(garrow_column_get_raw(column));
+    } else if (GARROW_IS_ARRAY(node->data)) {
+      auto array = GARROW_ARRAY(node->data);
+      arrow_arrays.push_back(garrow_array_get_raw(array));
+    } else if (GARROW_IS_RECORD_BATCH(node->data)) {
+      auto record_batch = GARROW_RECORD_BATCH(node->data);
+      arrow_record_batches.push_back(garrow_record_batch_get_raw(record_batch));
+    } else {
+      g_set_error(error,
+                  GARROW_ERROR,
+                  GARROW_ERROR_INVALID,
+                  "%s: %s",
+                  context,
+                  "value must be one of "
+                  "GArrowColumn, GArrowArray and GArrowRecordBatch");
+      return NULL;
+    }
+  }
+
+  size_t n_types = 0;
+  if (!arrow_columns.empty()) {
+    ++n_types;
+  }
+  if (!arrow_arrays.empty()) {
+    ++n_types;
+  }
+  if (!arrow_record_batches.empty()) {
+    ++n_types;
+  }
+  if (n_types > 1) {
+    g_set_error(error,
+                GARROW_ERROR,
+                GARROW_ERROR_INVALID,
+                "%s: %s",
+                context,
+                "all values must be the same objects of "
+                "GArrowColumn, GArrowArray or GArrowRecordBatch");
+    return NULL;
+  }
+
+  if (!arrow_columns.empty()) {
+    auto arrow_table = arrow::Table::Make(arrow_schema, arrow_columns);
+    auto status = arrow_table->Validate();
+    if (garrow_error_check(error, status, context)) {
+      return garrow_table_new_raw(&arrow_table);
+    } else {
+      return NULL;
+    }
+  } else if (!arrow_arrays.empty()) {
+    auto arrow_table = arrow::Table::Make(arrow_schema, arrow_arrays);
+    auto status = arrow_table->Validate();
+    if (garrow_error_check(error, status, context)) {
+      return garrow_table_new_raw(&arrow_table);
+    } else {
+      return NULL;
+    }
+  } else {
+    std::shared_ptr<arrow::Table> arrow_table;
+    auto status = arrow::Table::FromRecordBatches(arrow_schema,
+                                                  arrow_record_batches,
+                                                  &arrow_table);
+    if (garrow_error_check(error, status, context)) {
+      return garrow_table_new_raw(&arrow_table);
+    } else {
+      return NULL;
+    }
+  }
+}
+
+/**
+ * garrow_table_new_columns:
+ * @schema: The schema of the table.
+ * @columns: (array length=n_columns): The columns of the table.
+ * @n_columns: The number of columns.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowTable or %NULL on error.
+ *
+ * Since: 0.12.0
+ */
+GArrowTable *
+garrow_table_new_columns(GArrowSchema *schema,
+                         GArrowColumn **columns,
+                         gsize n_columns,
+                         GError **error)
+{
+  auto arrow_schema = garrow_schema_get_raw(schema);
+  std::vector<std::shared_ptr<arrow::Column>> arrow_columns;
+  for (gsize i = 0; i < n_columns; ++i) {
+    arrow_columns.push_back(garrow_column_get_raw(columns[i]));
+  }
+
+  auto arrow_table = arrow::Table::Make(arrow_schema, arrow_columns);
+  auto status = arrow_table->Validate();
+  if (garrow_error_check(error, status, "[table][new][columns]")) {
+    return garrow_table_new_raw(&arrow_table);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_table_new_arrays:
+ * @schema: The schema of the table.
+ * @arrays: (array length=n_arrays): The arrays of the table.
+ * @n_arrays: The number of arrays.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowTable or %NULL on error.
+ *
+ * Since: 0.12.0
+ */
+GArrowTable *
+garrow_table_new_arrays(GArrowSchema *schema,
+                        GArrowArray **arrays,
+                        gsize n_arrays,
+                        GError **error)
+{
+  auto arrow_schema = garrow_schema_get_raw(schema);
+  std::vector<std::shared_ptr<arrow::Array>> arrow_arrays;
+  for (gsize i = 0; i < n_arrays; ++i) {
+    arrow_arrays.push_back(garrow_array_get_raw(arrays[i]));
+  }
+
+  auto arrow_table = arrow::Table::Make(arrow_schema, arrow_arrays);
+  auto status = arrow_table->Validate();
+  if (garrow_error_check(error, status, "[table][new][arrays]")) {
+    return garrow_table_new_raw(&arrow_table);
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * garrow_table_new_record_batches:
+ * @schema: The schema of the table.
+ * @record_batches: (array length=n_record_batches): The record batches
+ *   that have data for the table.
+ * @n_record_batches: The number of record batches.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowTable or %NULL on error.
+ *
+ * Since: 0.12.0
+ */
+GArrowTable *
+garrow_table_new_record_batches(GArrowSchema *schema,
+                                GArrowRecordBatch **record_batches,
+                                gsize n_record_batches,
+                                GError **error)
+{
+  auto arrow_schema = garrow_schema_get_raw(schema);
+  std::vector<std::shared_ptr<arrow::RecordBatch>> arrow_record_batches;
+  for (gsize i = 0; i < n_record_batches; ++i) {
+    auto arrow_record_batch = garrow_record_batch_get_raw(record_batches[i]);
+    arrow_record_batches.push_back(arrow_record_batch);
+  }
+
+  std::shared_ptr<arrow::Table> arrow_table;
+  auto status = arrow::Table::FromRecordBatches(arrow_schema,
+                                                arrow_record_batches,
+                                                &arrow_table);
+  if (garrow_error_check(error, status, "[table][new][record-batches]")) {
+    return garrow_table_new_raw(&arrow_table);
+  } else {
+    return NULL;
+  }
+}
+
+/**
  * garrow_table_equal:
  * @table: A #GArrowTable.
  * @other_table: A #GArrowTable to be compared.
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index ef7b0f5..bde2535 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -20,7 +20,9 @@
 #pragma once
 
 #include <arrow-glib/column.h>
+#include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
+#include <arrow-glib/version.h>
 
 G_BEGIN_DECLS
 
@@ -35,8 +37,35 @@ struct _GArrowTableClass
   GObjectClass parent_class;
 };
 
-GArrowTable    *garrow_table_new           (GArrowSchema *schema,
-                                            GList *columns);
+#ifndef GARROW_DISABLE_DEPRECATED
+GARROW_DEPRECATED_IN_0_12_FOR(garrow_table_new_values)
+GArrowTable *
+garrow_table_new(GArrowSchema *schema,
+                 GList *columns);
+#endif
+GARROW_AVAILABLE_IN_0_12
+GArrowTable *
+garrow_table_new_values(GArrowSchema *schema,
+                        GList *values,
+                        GError **error);
+GARROW_AVAILABLE_IN_0_12
+GArrowTable *
+garrow_table_new_columns(GArrowSchema *schema,
+                         GArrowColumn **columns,
+                         gsize n_columns,
+                         GError **error);
+GARROW_AVAILABLE_IN_0_12
+GArrowTable *
+garrow_table_new_arrays(GArrowSchema *schema,
+                        GArrowArray **arrays,
+                        gsize n_arrays,
+                        GError **error);
+GARROW_AVAILABLE_IN_0_12
+GArrowTable *
+garrow_table_new_record_batches(GArrowSchema *schema,
+                                GArrowRecordBatch **record_batches,
+                                gsize n_record_batches,
+                                GError **error);
 
 gboolean        garrow_table_equal         (GArrowTable *table,
                                             GArrowTable *other_table);
diff --git a/c_glib/arrow-glib/version.h.in b/c_glib/arrow-glib/version.h.in
index eb73425..501827d 100644
--- a/c_glib/arrow-glib/version.h.in
+++ b/c_glib/arrow-glib/version.h.in
@@ -111,6 +111,15 @@
 #endif
 
 /**
+ * GARROW_VERSION_0_12:
+ *
+ * You can use this macro value for compile time API version check.
+ *
+ * Since: 0.12.0
+ */
+#define GARROW_VERSION_0_12 G_ENCODE_VERSION(0, 12)
+
+/**
  * GARROW_VERSION_0_10:
  *
  * You can use this macro value for compile time API version check.
@@ -166,6 +175,20 @@
 
 #define GARROW_AVAILABLE_IN_ALL
 
+#if GARROW_VERSION_MIN_REQUIRED >= GARROW_VERSION_0_12
+#  define GARROW_DEPRECATED_IN_0_12               GARROW_DEPRECATED
+#  define GARROW_DEPRECATED_IN_0_12_FOR(function) GARROW_DEPRECATED_FOR(function)
+#else
+#  define GARROW_DEPRECATED_IN_0_12
+#  define GARROW_DEPRECATED_IN_0_12_FOR(function)
+#endif
+
+#if GARROW_VERSION_MAX_ALLOWED < GARROW_VERSION_0_12
+#  define GARROW_AVAILABLE_IN_0_12 GARROW_UNAVAILABLE(0, 12)
+#else
+#  define GARROW_AVAILABLE_IN_0_12
+#endif
+
 #if GARROW_VERSION_MIN_REQUIRED >= GARROW_VERSION_0_10
 #  define GARROW_DEPRECATED_IN_0_10               GARROW_DEPRECATED
 #  define GARROW_DEPRECATED_IN_0_10_FOR(function) GARROW_DEPRECATED_FOR(function)
diff --git a/c_glib/test/test-table.rb b/c_glib/test/test-table.rb
index 4394ad1..871e0d7 100644
--- a/c_glib/test/test-table.rb
+++ b/c_glib/test/test-table.rb
@@ -17,21 +17,19 @@
 
 class TestTable < Test::Unit::TestCase
   include Helper::Buildable
+  include Helper::Omittable
 
   sub_test_case(".new") do
-    def test_columns
-      fields = [
+    def setup
+      @fields = [
         Arrow::Field.new("visible", Arrow::BooleanDataType.new),
         Arrow::Field.new("valid", Arrow::BooleanDataType.new),
       ]
-      schema = Arrow::Schema.new(fields)
-      columns = [
-        Arrow::Column.new(fields[0], build_boolean_array([true])),
-        Arrow::Column.new(fields[1], build_boolean_array([false])),
-      ]
-      table = Arrow::Table.new(schema, columns)
+      @schema = Arrow::Schema.new(@fields)
+    end
 
-      data = table.n_columns.times.collect do |i|
+    def dump_table(table)
+      table.n_columns.times.collect do |i|
         column = table.get_column(i)
         values = []
         column.data.chunks.each do |chunk|
@@ -44,11 +42,54 @@ class TestTable < Test::Unit::TestCase
           values,
         ]
       end
+    end
+
+    def test_columns
+      columns = [
+        Arrow::Column.new(@fields[0], build_boolean_array([true])),
+        Arrow::Column.new(@fields[1], build_boolean_array([false])),
+      ]
+      table = Arrow::Table.new(@schema, columns)
       assert_equal([
                      ["visible", [true]],
                      ["valid", [false]],
                    ],
-                   data)
+                   dump_table(table))
+    end
+
+    def test_arrays
+      require_gi_bindings(3, 3, 1)
+      arrays = [
+        build_boolean_array([true]),
+        build_boolean_array([false]),
+      ]
+      table = Arrow::Table.new(@schema, arrays)
+      assert_equal([
+                     ["visible", [true]],
+                     ["valid", [false]],
+                   ],
+                   dump_table(table))
+    end
+
+    def test_record_batches
+      require_gi_bindings(3, 3, 1)
+      record_batches = [
+        build_record_batch({
+                             "visible" => build_boolean_array([true]),
+                             "valid" => build_boolean_array([false])
+                           }),
+        build_record_batch({
+                             "visible" => build_boolean_array([false]),
+                             "valid" => build_boolean_array([true])
+                           }),
+      ]
+      table = Arrow::Table.new(@schema, record_batches)
+
+      assert_equal([
+                     ["visible", [true, false]],
+                     ["valid", [false, true]],
+                   ],
+                   dump_table(table))
     end
   end