You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2018/12/12 18:54:55 UTC
[arrow] branch master updated: ARROW-3470: [C++] Fix row-wise
example
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 7ddfba6 ARROW-3470: [C++] Fix row-wise example
7ddfba6 is described below
commit 7ddfba6693db99ec8ea38b6fd244c5d6e2af3295
Author: François Saint-Jacques <fs...@gmail.com>
AuthorDate: Wed Dec 12 19:54:45 2018 +0100
ARROW-3470: [C++] Fix row-wise example
- Implement the `ADD_EXAMPLE` cmake function with new ctest label
`example`, also covered by the `runexample` target. This can be
toggled via the `ARROW_BUILD_EXAMPLES` option which is ON by default.
- Implement fully working `row-wise-conversion-example.cc` and add it to
the default build.
- Update documentation to embed (manually) the newly created example.
Author: François Saint-Jacques <fs...@gmail.com>
Closes #3078 from fsaintjacques/ARROW-3470-out-of-date-example and squashes the following commits:
fab63f6f <François Saint-Jacques> ARROW-3470: Fix status macro
1eba067d <François Saint-Jacques> ARROW-3470: Fix row-wise example
---
ci/appveyor-cpp-build.bat | 3 +
ci/appveyor-cpp-test-cmake-script.bat | 8 +
ci/cpp-msvc-build-main.bat | 1 +
ci/travis_before_script_cpp.sh | 1 +
cpp/CMakeLists.txt | 15 ++
cpp/apidoc/tutorials/row_wise_conversion.md | 194 ----------------------
cpp/cmake_modules/BuildUtils.cmake | 60 +++++++
cpp/examples/arrow/CMakeLists.txt | 18 ++
cpp/examples/arrow/row-wise-conversion-example.cc | 190 +++++++++++++++++++++
cpp/src/arrow/status.h | 4 +-
docs/source/cpp/{index.rst => examples.rst} | 19 +--
docs/source/cpp/index.rst | 1 +
12 files changed, 308 insertions(+), 206 deletions(-)
diff --git a/ci/appveyor-cpp-build.bat b/ci/appveyor-cpp-build.bat
index d20a021..387dd55 100644
--- a/ci/appveyor-cpp-build.bat
+++ b/ci/appveyor-cpp-build.bat
@@ -35,6 +35,7 @@ if "%JOB%" == "Static_Crt_Build" (
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=Debug ^
-DARROW_TEST_LINKAGE=static ^
-DARROW_CXXFLAGS="/MP" ^
@@ -53,6 +54,7 @@ if "%JOB%" == "Static_Crt_Build" (
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=Release ^
-DARROW_TEST_LINKAGE=static ^
-DCMAKE_CXX_FLAGS_RELEASE="/MT %CMAKE_CXX_FLAGS_RELEASE%" ^
@@ -79,6 +81,7 @@ if "%JOB%" == "Build_Debug" (
-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_BUILD_STATIC=OFF ^
-DARROW_CXXFLAGS="/MP" ^
diff --git a/ci/appveyor-cpp-test-cmake-script.bat b/ci/appveyor-cpp-test-cmake-script.bat
index 8158a44..415406c 100644
--- a/ci/appveyor-cpp-test-cmake-script.bat
+++ b/ci/appveyor-cpp-test-cmake-script.bat
@@ -33,6 +33,7 @@ set FLATBUFFERS_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -51,6 +52,7 @@ set GFLAGS_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -69,6 +71,7 @@ set SNAPPY_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -87,6 +90,7 @@ set ZLIB_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -105,6 +109,7 @@ set BROTLI_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -123,6 +128,7 @@ set LZ4_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -141,6 +147,7 @@ set ZSTD_HOME=WrongPath
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. >nul 2>error.txt
@@ -166,6 +173,7 @@ set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
cmake -G "%GENERATOR%" ^
-DARROW_BOOST_USE_SHARED=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_CXXFLAGS="/MP" ^
.. 2>output.txt
diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat
index 560f504..6441707 100644
--- a/ci/cpp-msvc-build-main.bat
+++ b/ci/cpp-msvc-build-main.bat
@@ -49,6 +49,7 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DARROW_BUILD_STATIC=OFF ^
-DARROW_BUILD_TESTS=ON ^
+ -DARROW_BUILD_EXAMPLES=ON ^
-DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
-DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^
-DARROW_PARQUET=ON ^
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 6465f28..a77fcd8 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -67,6 +67,7 @@ else
$CMAKE_COMMON_FLAGS \
-DARROW_BUILD_BENCHMARKS=ON \
-DARROW_BUILD_TESTS=ON \
+-DARROW_BUILD_EXAMPLES=ON \
-DARROW_BUILD_UTILITIES=OFF \
-DARROW_INSTALL_NAME_RPATH=OFF"
fi
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 35707de..a83b9dd 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -122,6 +122,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
"Build the Arrow micro benchmarks, default OFF"
OFF)
+ option(ARROW_BUILD_EXAMPLES
+ "Build the Arrow examples, default OFF"
+ OFF)
+
set(ARROW_TEST_LINKAGE "shared" CACHE STRING
"Linkage of Arrow libraries with unit tests executables. \
static|shared (default shared)")
@@ -447,6 +451,10 @@ if(NOT ARROW_BUILD_BENCHMARKS)
set(NO_BENCHMARKS 1)
endif()
+if(NOT ARROW_BUILD_EXAMPLES)
+ set(NO_EXAMPLES 1)
+endif()
+
if (NOT ARROW_FUZZING)
set(NO_FUZZING 1)
endif()
@@ -735,12 +743,14 @@ pass ARROW_BUILD_SHARED=on")
endif()
# Use shared linking for unit tests if it's available
set(ARROW_TEST_LINK_LIBS ${ARROW_TEST_SHARED_LINK_LIBS})
+ set(ARROW_EXAMPLE_LINK_LIBS arrow_shared)
else()
if (NOT ARROW_BUILD_STATIC)
message(FATAL_ERROR "If using static linkage for unit tests, must also \
pass ARROW_BUILD_STATIC=on")
endif()
set(ARROW_TEST_LINK_LIBS ${ARROW_TEST_STATIC_LINK_LIBS})
+ set(ARROW_EXAMPLE_LINK_LIBS arrow_static)
endif()
if (ARROW_BUILD_BENCHMARKS)
@@ -805,6 +815,11 @@ if(ARROW_GANDIVA)
add_subdirectory(src/gandiva)
endif()
+if(ARROW_BUILD_EXAMPLES)
+ add_custom_target(runexample ctest -L example)
+ add_subdirectory(examples/arrow)
+endif()
+
include(CMakePackageConfigHelpers)
# Makes the project importable from the build directory
diff --git a/cpp/apidoc/tutorials/row_wise_conversion.md b/cpp/apidoc/tutorials/row_wise_conversion.md
deleted file mode 100644
index 750a923..0000000
--- a/cpp/apidoc/tutorials/row_wise_conversion.md
+++ /dev/null
@@ -1,194 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-Convert a vector of row-wise data into an Arrow table
-=====================================================
-
-While we want to use columnar data structures to build efficient operations, we
-often receive data in a row-wise fashion from other systems. In the following,
-we want give a brief introduction into the classes provided by Apache Arrow by
-showing how to transform row-wise data into a columnar table.
-
-The data in this example is stored in the following struct:
-
-```
-struct data_row {
- int64_t id;
- double cost;
- std::vector<double> cost_components;
-};
-
-std::vector<data_row> rows;
-```
-
-The final representation should be an `arrow::Table` which in turn is made up of
-an `arrow::Schema` and a list of `arrow::Column`. An `arrow::Column` is again a
-named collection of one or more `arrow::Array` instances. As the first step, we
-will iterate over the data and build up the arrays incrementally. For this task,
-we provide `arrow::ArrayBuilder` classes that help in the construction of the
-final `arrow::Array` instances.
-
-For each type, Arrow has a specially typed builder class. For the primitive
-values `id` and `cost` we can use the respective `arrow::Int64Builder` and
-`arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two
-builders, a top-level `arrow::ListBuilder` that builds the array of offsets and
-a nested `arrow::DoubleBuilder` that constructs the underlying values array that
-is referenced by the offsets in the former array.
-
-```
-// The builders are more efficient using
-// arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of
-// the underlying memory regions in-place. At the moment, arrow::jemalloc is only
-// supported on Unix systems, not Windows.
-
-using arrow::DoubleBuilder;
-using arrow::Int64Builder;
-using arrow::ListBuilder;
-
-arrow::MemoryPool* pool = arrow::default_memory_pool();
-Int64Builder id_builder(pool);
-DoubleBuilder cost_builder(pool);
-std::unique_ptr<DoubleBuilder> components_values_builder(new DoubleBuilder(pool));
-ListBuilder components_builder(pool, std::move(components_values_builder));
-```
-
-Now we can loop over our existing data and insert it into the builders. The
-`Append` calls here may fail (e.g. we cannot allocate enough additional memory).
-Thus we need to check their return values. For more information on these values,
-check the documentation about `arrow::Status`.
-
-```
-for (const data_row& row : rows) {
- ARROW_RETURN_NOT_OK(id_builder.Append(row.id));
- ARROW_RETURN_NOT_OK(cost_builder.Append(row.cost));
-
- // Indicate the start of a new list row. This will memorise the current
- // offset in the values builder.
- ARROW_RETURN_NOT_OK(components_builder.Append());
- // Store the actual values. The final nullptr argument tells the underyling
- // builder that all added values are valid, i.e. non-null.
- ARROW_RETURN_NOT_OK(components_values_builder->Append(
- row.cost_components.data(), row.cost_components.size(),
- nullptr);
-}
-```
-
-At the end, we finalise the arrays, declare the (type) schema and combine them
- into a single `arrow::Table`:
-
-```
-std::shared_ptr<arrow::Array> id_array;
-ARROW_RETURN_NOT_OK(id_builder.Finish(&id_array));
-std::shared_ptr<arrow::Array> cost_array;
-ARROW_RETURN_NOT_OK(cost_builder.Finish(&cost_array));
-std::shared_ptr<arrow::Array> cost_components_array;
-ARROW_RETURN_NOT_OK(components_builder.Finish(&cost_components_array));
-
-std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
- arrow::field("id", arrow::int64()),
- arrow::field("cost", arrow::float64()),
- arrow::field("cost_components", arrow::list(arrow::float64()))
-};
-auto schema = std::make_shared<arrow::Schema>(schema_vector);
-
-std::shared_ptr<arrow::Table> table = arrow::Table::Make(schema,
- {id_array, cost_array, cost_components_array});
-```
-
-The final `table` variable is the one we then can pass on to other functions
-that can consume Apache Arrow memory structures. This object has ownership of
-all referenced data, thus we don't have to care about undefined references once
-we leave the scope of the function building the table and its underlying arrays.
-
-<!-- TODO: Add an example with nullable entries -->
-
-Converting an Arrow Table back into row-wise representation
-===========================================================
-
-To convert an Arrow table back into the same row-wise representation as in the
-above section, we first will check that the table conforms to our expected
-schema and then will build up the vector of rows incrementally.
-
-For the check if the table is as expected, we can utilise solely its schema.
-
-```
-// This is our input that was passed in from the outside.
-std::shared_ptr<arrow::Table> table;
-
-std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
- arrow::field("id", arrow::int64()),
- arrow::field("cost", arrow::float64()),
- arrow::field("cost_components", arrow::list(arrow::float64()))
-};
-auto expected_schema = std::make_shared<arrow::Schema>(schema_vector);
-
-if (!expected_schema->Equals(*table->schema())) {
- // The table doesn't have the expected schema thus we cannot directly
- // convert it to our target representation.
- // TODO: Implement your custom error handling logic here.
-}
-```
-
-As we have ensured that the table has the expected structure, we can unpack the
-underlying arrays. For the primitive columns `id` and `cost` we can use the high
-level functions to get the values whereas for the nested column
-`cost_components` we need to access the C-pointer to the data to copy its
-contents into the resulting `std::vector<double>`. Here we need to be care to
-also add the offset to the pointer. This offset is needed to enable zero-copy
-slicing operations. While this could be adjusted automatically for double
-arrays, this cannot be done for the accompanying bitmap as often the slicing
-border would be inside a byte.
-
-```
-// For simplicity, we assume that all arrays consist of a single chunk here.
-// In a productive implementation this should either be explicitly check or code
-// added that can treat chunked arrays.
-
-auto ids = std::static_pointer_cast<arrow::Int64Array>(
- table->column(0)->data()->chunk(0));
-auto costs = std::static_pointer_cast<arrow::DoubleArray(
- table->column(1)->data()->chunk(0));
-auto cost_components = std::static_pointer_cast<arrow::ListArray(
- table->column(2)->data()->chunk(0));
-auto cost_components_values = std::static_pointer_cast<arrow::DoubleArray>(
- cost_components->values());
-// To enable zero-copy slices, the native values pointer might need to account
-// for this slicing offset. This is not needed for the higher level functions
-// like Value(…) that already account for this offset internally.
-const double* cost_components_values_ptr = cost_components_values->data()
- + cost_components_values->offset();
-```
-
-After we have unpacked the arrays from the table, we can iterate over them in a
-row-wise fashion and fill our target, row-wise representation.
-
-```
-std::vector<data_row> rows;
-
-for (int64_t i = 0; i < table->num_rows(); i++) {
- // Another simplification in this example is that we assume that there are
- // no null entries, e.g. each row is fill with valid values.
- int64_t id = ids->Value(i);
- double cost = costs->Value(i);
- const double* first = cost_components_values_ptr + cost_components->value_offset(i);
- const double* last = cost_components_values_ptr + cost_components->value_offset(i + 1);
- std::vector<double> components_vec(first, last);
- rows.push_back({id, cost, components_vec});
-}
-```
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
index bcf6728..d5978e1 100644
--- a/cpp/cmake_modules/BuildUtils.cmake
+++ b/cpp/cmake_modules/BuildUtils.cmake
@@ -481,6 +481,66 @@ function(ADD_TEST_CASE REL_TEST_NAME)
endfunction()
############################################################
+# Examples
+############################################################
+# Add a new example, with or without an executable that should be built.
+# If examples are enabled then they will be run along side unit tests with ctest.
+# 'make runexample' to build/run only examples.
+#
+# REL_EXAMPLE_NAME is the name of the example app. It may be a single component
+# (e.g. monotime-example) or contain additional components (e.g.
+# net/net_util-example). Either way, the last component must be a globally
+# unique name.
+
+# The example will registered as unit test with ctest with a label
+# of 'example'.
+#
+# Arguments after the test name will be passed to set_tests_properties().
+#
+# \arg PREFIX a string to append to the name of the example executable. For
+# example, if you have src/arrow/foo/bar-example.cc, then PREFIX "foo" will
+# create test executable foo-bar-example
+function(ADD_ARROW_EXAMPLE REL_EXAMPLE_NAME)
+ set(options)
+ set(one_value_args)
+ set(multi_value_args EXTRA_LINK_LIBS DEPENDENCIES PREFIX)
+ cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
+ if(ARG_UNPARSED_ARGUMENTS)
+ message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+ endif()
+
+ if(NO_EXAMPLES)
+ return()
+ endif()
+ get_filename_component(EXAMPLE_NAME ${REL_EXAMPLE_NAME} NAME_WE)
+
+ if(ARG_PREFIX)
+ set(EXAMPLE_NAME "${ARG_PREFIX}-${EXAMPLE_NAME}")
+ endif()
+
+ if(EXISTS ${CMAKE_SOURCE_DIR}/examples/arrow/${REL_EXAMPLE_NAME}.cc)
+ # This example has a corresponding .cc file, set it up as an executable.
+ set(EXAMPLE_PATH "${EXECUTABLE_OUTPUT_PATH}/${EXAMPLE_NAME}")
+ add_executable(${EXAMPLE_NAME} "${REL_EXAMPLE_NAME}.cc")
+ target_link_libraries(${EXAMPLE_NAME} ${ARROW_EXAMPLE_LINK_LIBS})
+ add_dependencies(runexample ${EXAMPLE_NAME})
+ set(NO_COLOR "--color_print=false")
+
+ if (ARG_EXTRA_LINK_LIBS)
+ target_link_libraries(${EXAMPLE_NAME} ${ARG_EXTRA_LINK_LIBS})
+ endif()
+ endif()
+
+ if (ARG_DEPENDENCIES)
+ add_dependencies(${EXAMPLE_NAME} ${ARG_DEPENDENCIES})
+ endif()
+
+
+ add_test(${EXAMPLE_NAME} ${EXAMPLE_PATH})
+ set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example")
+endfunction()
+
+############################################################
# Fuzzing
############################################################
# Add new fuzzing test executable.
diff --git a/cpp/examples/arrow/CMakeLists.txt b/cpp/examples/arrow/CMakeLists.txt
new file mode 100644
index 0000000..6ecb537
--- /dev/null
+++ b/cpp/examples/arrow/CMakeLists.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ADD_ARROW_EXAMPLE(row-wise-conversion-example)
diff --git a/cpp/examples/arrow/row-wise-conversion-example.cc b/cpp/examples/arrow/row-wise-conversion-example.cc
new file mode 100644
index 0000000..db8c287
--- /dev/null
+++ b/cpp/examples/arrow/row-wise-conversion-example.cc
@@ -0,0 +1,190 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <iostream>
+#include <vector>
+
+#include <arrow/api.h>
+
+using arrow::DoubleBuilder;
+using arrow::Int64Builder;
+using arrow::ListBuilder;
+
+// While we want to use columnar data structures to build efficient operations, we
+// often receive data in a row-wise fashion from other systems. In the following,
+// we want give a brief introduction into the classes provided by Apache Arrow by
+// showing how to transform row-wise data into a columnar table.
+//
+// The data in this example is stored in the following struct:
+struct data_row {
+ int64_t id;
+ double cost;
+ std::vector<double> cost_components;
+};
+
+// Transforming a vector of structs into a columnar Table.
+//
+// The final representation should be an `arrow::Table` which in turn is made up of
+// an `arrow::Schema` and a list of `arrow::Column`. An `arrow::Column` is again a
+// named collection of one or more `arrow::Array` instances. As the first step, we
+// will iterate over the data and build up the arrays incrementally. For this task,
+// we provide `arrow::ArrayBuilder` classes that help in the construction of the
+// final `arrow::Array` instances.
+//
+// For each type, Arrow has a specially typed builder class. For the primitive
+// values `id` and `cost` we can use the respective `arrow::Int64Builder` and
+// `arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two
+// builders, a top-level `arrow::ListBuilder` that builds the array of offsets and
+// a nested `arrow::DoubleBuilder` that constructs the underlying values array that
+// is referenced by the offsets in the former array.
+arrow::Status VectorToColumnarTable(const std::vector<struct data_row>& rows,
+ std::shared_ptr<arrow::Table>* table) {
+ // The builders are more efficient using
+ // arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of
+ // the underlying memory regions in-place. At the moment, arrow::jemalloc is only
+ // supported on Unix systems, not Windows.
+ arrow::MemoryPool* pool = arrow::default_memory_pool();
+
+ Int64Builder id_builder(pool);
+ DoubleBuilder cost_builder(pool);
+ ListBuilder components_builder(pool, std::make_shared<DoubleBuilder>(pool));
+ // The following builder is owned by components_builder.
+ DoubleBuilder& cost_components_builder =
+ *(static_cast<DoubleBuilder*>(components_builder.value_builder()));
+
+ // Now we can loop over our existing data and insert it into the builders. The
+ // `Append` calls here may fail (e.g. we cannot allocate enough additional memory).
+ // Thus we need to check their return values. For more information on these values,
+ // check the documentation about `arrow::Status`.
+ for (const data_row& row : rows) {
+ ARROW_RETURN_NOT_OK(id_builder.Append(row.id));
+ ARROW_RETURN_NOT_OK(cost_builder.Append(row.cost));
+
+ // Indicate the start of a new list row. This will memorise the current
+ // offset in the values builder.
+ ARROW_RETURN_NOT_OK(components_builder.Append());
+ // Store the actual values. The final nullptr argument tells the underyling
+ // builder that all added values are valid, i.e. non-null.
+ ARROW_RETURN_NOT_OK(cost_components_builder.AppendValues(row.cost_components.data(),
+ row.cost_components.size()));
+ }
+
+ // At the end, we finalise the arrays, declare the (type) schema and combine them
+ // into a single `arrow::Table`:
+ std::shared_ptr<arrow::Array> id_array;
+ ARROW_RETURN_NOT_OK(id_builder.Finish(&id_array));
+ std::shared_ptr<arrow::Array> cost_array;
+ ARROW_RETURN_NOT_OK(cost_builder.Finish(&cost_array));
+ // No need to invoke cost_components_builder.Finish because it is implied by
+ // the parent builder's Finish invocation.
+ std::shared_ptr<arrow::Array> cost_components_array;
+ ARROW_RETURN_NOT_OK(components_builder.Finish(&cost_components_array));
+
+ std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
+ arrow::field("id", arrow::int64()), arrow::field("cost", arrow::float64()),
+ arrow::field("cost_components", arrow::list(arrow::float64()))};
+
+ auto schema = std::make_shared<arrow::Schema>(schema_vector);
+
+ // The final `table` variable is the one we then can pass on to other functions
+ // that can consume Apache Arrow memory structures. This object has ownership of
+ // all referenced data, thus we don't have to care about undefined references once
+ // we leave the scope of the function building the table and its underlying arrays.
+ *table = arrow::Table::Make(schema, {id_array, cost_array, cost_components_array});
+
+ return arrow::Status::OK();
+}
+
+arrow::Status ColumnarTableToVector(const std::shared_ptr<arrow::Table>& table,
+ std::vector<struct data_row>* rows) {
+ // To convert an Arrow table back into the same row-wise representation as in the
+ // above section, we first will check that the table conforms to our expected
+ // schema and then will build up the vector of rows incrementally.
+ //
+ // For the check if the table is as expected, we can utilise solely its schema.
+ std::vector<std::shared_ptr<arrow::Field>> schema_vector = {
+ arrow::field("id", arrow::int64()), arrow::field("cost", arrow::float64()),
+ arrow::field("cost_components", arrow::list(arrow::float64()))};
+ auto expected_schema = std::make_shared<arrow::Schema>(schema_vector);
+
+ if (!expected_schema->Equals(*table->schema())) {
+ // The table doesn't have the expected schema thus we cannot directly
+ // convert it to our target representation.
+ return arrow::Status::Invalid("Schemas are not matching!");
+ }
+
+ // As we have ensured that the table has the expected structure, we can unpack the
+ // underlying arrays. For the primitive columns `id` and `cost` we can use the high
+ // level functions to get the values whereas for the nested column
+ // `cost_components` we need to access the C-pointer to the data to copy its
+ // contents into the resulting `std::vector<double>`. Here we need to be care to
+ // also add the offset to the pointer. This offset is needed to enable zero-copy
+ // slicing operations. While this could be adjusted automatically for double
+ // arrays, this cannot be done for the accompanying bitmap as often the slicing
+ // border would be inside a byte.
+
+ auto ids =
+ std::static_pointer_cast<arrow::Int64Array>(table->column(0)->data()->chunk(0));
+ auto costs =
+ std::static_pointer_cast<arrow::DoubleArray>(table->column(1)->data()->chunk(0));
+ auto cost_components =
+ std::static_pointer_cast<arrow::ListArray>(table->column(2)->data()->chunk(0));
+ auto cost_components_values =
+ std::static_pointer_cast<arrow::DoubleArray>(cost_components->values());
+ // To enable zero-copy slices, the native values pointer might need to account
+ // for this slicing offset. This is not needed for the higher level functions
+ // like Value(…) that already account for this offset internally.
+ const double* ccv_ptr = cost_components_values->data()->GetValues<double>(1);
+
+ for (int64_t i = 0; i < table->num_rows(); i++) {
+ // Another simplification in this example is that we assume that there are
+ // no null entries, e.g. each row is fill with valid values.
+ int64_t id = ids->Value(i);
+ double cost = costs->Value(i);
+ const double* first = ccv_ptr + cost_components->value_offset(i);
+ const double* last = ccv_ptr + cost_components->value_offset(i + 1);
+ std::vector<double> components_vec(first, last);
+ rows->push_back({id, cost, components_vec});
+ }
+
+ return arrow::Status::OK();
+}
+
+#define EXIT_ON_FAILURE(expr) \
+ do { \
+ arrow::Status status_ = (expr); \
+ if (!status_.ok()) { \
+ std::cerr << status_.message() << std::endl; \
+ return EXIT_FAILURE; \
+ } \
+ } while (0);
+
+int main(int argc, char** argv) {
+ std::vector<data_row> rows = {
+ {1, 1.0, {1.0}}, {2, 2.0, {1.0, 2.0}}, {3, 3.0, {1.0, 2.0, 3.0}}};
+
+ std::shared_ptr<arrow::Table> table;
+ EXIT_ON_FAILURE(VectorToColumnarTable(rows, &table));
+
+ std::vector<data_row> expected_rows;
+ EXIT_ON_FAILURE(ColumnarTableToVector(table, &expected_rows));
+
+ assert(rows.size() == expected_rows.size());
+
+ return EXIT_SUCCESS;
+}
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index ddf3d7e..e3632a6 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -36,7 +36,7 @@
if (ARROW_PREDICT_FALSE(!_s.ok())) { \
std::stringstream ss; \
ss << __FILE__ << ":" << __LINE__ << " code: " << #s << "\n" << _s.message(); \
- return Status(_s.code(), ss.str()); \
+ return ::arrow::Status(_s.code(), ss.str()); \
} \
} while (0)
@@ -69,7 +69,7 @@
std::stringstream ss; \
ss << __FILE__ << ":" << __LINE__ << " code: " << _status.CodeAsString() << " \n " \
<< _status.message(); \
- return Status(_status.code(), ss.str()); \
+ return ::arrow::Status(_status.code(), ss.str()); \
} \
} while (0)
diff --git a/docs/source/cpp/index.rst b/docs/source/cpp/examples.rst
similarity index 70%
copy from docs/source/cpp/index.rst
copy to docs/source/cpp/examples.rst
index 63290be..5f4372f 100644
--- a/docs/source/cpp/index.rst
+++ b/docs/source/cpp/examples.rst
@@ -15,17 +15,16 @@
.. specific language governing permissions and limitations
.. under the License.
-C++ Implementation
-==================
+.. default-domain:: cpp
+.. highlight:: cpp
-.. toctree::
- :maxdepth: 2
+Examples
+========
- getting_started
- api
+Row to columnar conversion
+--------------------------
-.. TODO add "topics" chapter
-.. - nested arrays
-.. - dictionary encoding
+The following example converts an array of structs to a :class:`arrow::Table`
+instance, and then converts it back to the original array of structs.
-.. TODO add "building" or "development" chapter
+.. literalinclude:: ../../../cpp/examples/arrow/row-wise-conversion-example.cc
diff --git a/docs/source/cpp/index.rst b/docs/source/cpp/index.rst
index 63290be..1d70e6a 100644
--- a/docs/source/cpp/index.rst
+++ b/docs/source/cpp/index.rst
@@ -22,6 +22,7 @@ C++ Implementation
:maxdepth: 2
getting_started
+ examples
api
.. TODO add "topics" chapter