You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2016/12/12 22:17:50 UTC
[1/5] arrow git commit: ARROW-418: [C++] Array / Builder class code
reorganization, flattening
Repository: arrow
Updated Branches:
refs/heads/master 73fe55683 -> 2c10d7cce
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/python/src/pyarrow/common.h
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h
index 7f3131e..7e33826 100644
--- a/python/src/pyarrow/common.h
+++ b/python/src/pyarrow/common.h
@@ -19,10 +19,11 @@
#define PYARROW_COMMON_H
#include "pyarrow/config.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/macros.h"
#include "pyarrow/visibility.h"
+#include "arrow/buffer.h"
+#include "arrow/util/macros.h"
+
namespace arrow { class MemoryPool; }
namespace pyarrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/python/src/pyarrow/io.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/io.cc b/python/src/pyarrow/io.cc
index e6dbc12..12f5ba0 100644
--- a/python/src/pyarrow/io.cc
+++ b/python/src/pyarrow/io.cc
@@ -21,8 +21,8 @@
#include <cstdlib>
#include "arrow/io/memory.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
#include "pyarrow/common.h"
[3/5] arrow git commit: ARROW-418: [C++] Array / Builder class code
reorganization, flattening
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/status.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
new file mode 100644
index 0000000..05f5b74
--- /dev/null
+++ b/cpp/src/arrow/status.h
@@ -0,0 +1,192 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// A Status encapsulates the result of an operation. It may indicate success,
+// or it may indicate an error with an associated error message.
+//
+// Multiple threads can invoke const methods on a Status without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same Status must use
+// external synchronization.
+
+// Adapted from Kudu github.com/cloudera/kudu
+
+#ifndef ARROW_STATUS_H_
+#define ARROW_STATUS_H_
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+#include "arrow/util/visibility.h"
+
+// Return the given status if it is not OK.
+#define ARROW_RETURN_NOT_OK(s) \
+ do { \
+ ::arrow::Status _s = (s); \
+ if (!_s.ok()) { return _s; } \
+ } while (0);
+
+// Return the given status if it is not OK, but first clone it and
+// prepend the given message.
+#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) \
+ do { \
+ ::arrow::Status _s = (s); \
+ if (::gutil::PREDICT_FALSE(!_s.ok())) return _s.CloneAndPrepend(msg); \
+ } while (0);
+
+// Return 'to_return' if 'to_call' returns a bad status.
+// The substitution for 'to_return' may reference the variable
+// 's' for the bad status.
+#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) \
+ do { \
+ ::arrow::Status s = (to_call); \
+ if (::gutil::PREDICT_FALSE(!s.ok())) return (to_return); \
+ } while (0);
+
+// If 'to_call' returns a bad status, CHECK immediately with a logged message
+// of 'msg' followed by the status.
+#define ARROW_CHECK_OK_PREPEND(to_call, msg) \
+ do { \
+ ::arrow::Status _s = (to_call); \
+ ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \
+ } while (0);
+
+// If the status is bad, CHECK immediately, appending the status to the
+// logged message.
+#define ARROW_CHECK_OK(s) ARROW_CHECK_OK_PREPEND(s, "Bad status")
+
+namespace arrow {
+
+#define RETURN_NOT_OK(s) \
+ do { \
+ Status _s = (s); \
+ if (!_s.ok()) { return _s; } \
+ } while (0);
+
+#define RETURN_NOT_OK_ELSE(s, else_) \
+ do { \
+ Status _s = (s); \
+ if (!_s.ok()) { \
+ else_; \
+ return _s; \
+ } \
+ } while (0);
+
+enum class StatusCode : char {
+ OK = 0,
+ OutOfMemory = 1,
+ KeyError = 2,
+ TypeError = 3,
+ Invalid = 4,
+ IOError = 5,
+ UnknownError = 9,
+ NotImplemented = 10,
+};
+
+class ARROW_EXPORT Status {
+ public:
+ // Create a success status.
+ Status() : state_(NULL) {}
+ ~Status() { delete[] state_; }
+
+ // Copy the specified status.
+ Status(const Status& s);
+ void operator=(const Status& s);
+
+ // Return a success status.
+ static Status OK() { return Status(); }
+
+ // Return error status of an appropriate type.
+ static Status OutOfMemory(const std::string& msg, int16_t posix_code = -1) {
+ return Status(StatusCode::OutOfMemory, msg, posix_code);
+ }
+
+ static Status KeyError(const std::string& msg) {
+ return Status(StatusCode::KeyError, msg, -1);
+ }
+
+ static Status TypeError(const std::string& msg) {
+ return Status(StatusCode::TypeError, msg, -1);
+ }
+
+ static Status UnknownError(const std::string& msg) {
+ return Status(StatusCode::UnknownError, msg, -1);
+ }
+
+ static Status NotImplemented(const std::string& msg) {
+ return Status(StatusCode::NotImplemented, msg, -1);
+ }
+
+ static Status Invalid(const std::string& msg) {
+ return Status(StatusCode::Invalid, msg, -1);
+ }
+
+ static Status IOError(const std::string& msg) {
+ return Status(StatusCode::IOError, msg, -1);
+ }
+
+ // Returns true iff the status indicates success.
+ bool ok() const { return (state_ == NULL); }
+
+ bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
+ bool IsKeyError() const { return code() == StatusCode::KeyError; }
+ bool IsInvalid() const { return code() == StatusCode::Invalid; }
+ bool IsIOError() const { return code() == StatusCode::IOError; }
+
+ bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
+ bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
+
+ // Return a string representation of this status suitable for printing.
+ // Returns the string "OK" for success.
+ std::string ToString() const;
+
+ // Return a string representation of the status code, without the message
+ // text or posix code information.
+ std::string CodeAsString() const;
+
+ // Get the POSIX code associated with this Status, or -1 if there is none.
+ int16_t posix_code() const;
+
+ StatusCode code() const {
+ return ((state_ == NULL) ? StatusCode::OK : static_cast<StatusCode>(state_[4]));
+ }
+
+ std::string message() const {
+ uint32_t length;
+ memcpy(&length, state_, sizeof(length));
+ std::string msg;
+ msg.append((state_ + 7), length);
+ return msg;
+ }
+
+ private:
+ // OK status has a NULL state_. Otherwise, state_ is a new[] array
+ // of the following form:
+ // state_[0..3] == length of message
+ // state_[4] == code
+ // state_[5..6] == posix_code
+ // state_[7..] == message
+ const char* state_;
+
+ Status(StatusCode code, const std::string& msg, int16_t posix_code);
+ static const char* CopyState(const char* s);
+};
+
+inline Status::Status(const Status& s) {
+ state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+}
+
+inline void Status::operator=(const Status& s) {
+ // The following condition catches both aliasing (when this == &s),
+ // and the common case where both s and *this are ok.
+ if (state_ != s.state_) {
+ delete[] state_;
+ state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+ }
+}
+
+} // namespace arrow
+
+#endif // ARROW_STATUS_H_
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/table-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/table-test.cc b/cpp/src/arrow/table-test.cc
index 743fb66..f62336d 100644
--- a/cpp/src/arrow/table-test.cc
+++ b/cpp/src/arrow/table-test.cc
@@ -21,13 +21,13 @@
#include "gtest/gtest.h"
+#include "arrow/array.h"
#include "arrow/column.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/status.h"
using std::shared_ptr;
using std::vector;
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/table.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/table.cc b/cpp/src/arrow/table.cc
index eb1258a..855d4ec 100644
--- a/cpp/src/arrow/table.cc
+++ b/cpp/src/arrow/table.cc
@@ -24,7 +24,7 @@
#include "arrow/array.h"
#include "arrow/column.h"
#include "arrow/schema.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/test-util.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index b86a180..aa310b1 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -28,17 +28,18 @@
#include "gtest/gtest.h"
#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
#include "arrow/column.h"
+#include "arrow/memory_pool.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
#include "arrow/util/logging.h"
-#include "arrow/util/memory-pool.h"
#include "arrow/util/random.h"
-#include "arrow/util/status.h"
#define ASSERT_RAISES(ENUM, expr) \
do { \
@@ -253,8 +254,9 @@ Status MakeRandomBytePoolBuffer(int32_t length, MemoryPool* pool,
} // namespace test
template <typename TYPE, typename C_TYPE>
-void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& is_valid,
- const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
+void ArrayFromVector(const std::shared_ptr<DataType>& type,
+ const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
+ std::shared_ptr<Array>* out) {
std::shared_ptr<Buffer> values_buffer;
std::shared_ptr<Buffer> values_bitmap;
@@ -272,6 +274,37 @@ void MakeArray(const std::shared_ptr<DataType>& type, const std::vector<bool>& i
values_buffer, null_count, values_bitmap);
}
+class TestBuilder : public ::testing::Test {
+ public:
+ void SetUp() {
+ pool_ = default_memory_pool();
+ type_ = TypePtr(new UInt8Type());
+ builder_.reset(new UInt8Builder(pool_, type_));
+ builder_nn_.reset(new UInt8Builder(pool_, type_));
+ }
+
+ protected:
+ MemoryPool* pool_;
+
+ TypePtr type_;
+ std::unique_ptr<ArrayBuilder> builder_;
+ std::unique_ptr<ArrayBuilder> builder_nn_;
+};
+
+template <class T, class Builder>
+Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
+ int size, Builder* builder, ArrayPtr* out) {
+ // Append the first 1000
+ for (int i = 0; i < size; ++i) {
+ if (valid_bytes[i] > 0) {
+ RETURN_NOT_OK(builder->Append(values[i]));
+ } else {
+ RETURN_NOT_OK(builder->AppendNull());
+ }
+ }
+ return builder->Finish(out);
+}
+
} // namespace arrow
#endif // ARROW_TEST_UTIL_H_
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/type.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 75f5086..5b172e4 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -20,7 +20,7 @@
#include <sstream>
#include <string>
-#include "arrow/util/status.h"
+#include "arrow/status.h"
namespace arrow {
@@ -220,6 +220,12 @@ std::vector<BufferDescr> UnionType::GetBufferLayout() const {
}
}
+std::string DecimalType::ToString() const {
+ std::stringstream s;
+ s << "decimal(" << precision << ", " << scale << ")";
+ return s.str();
+}
+
std::vector<BufferDescr> DecimalType::GetBufferLayout() const {
// TODO(wesm)
return {};
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/type.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 966706c..8637081 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -23,9 +23,9 @@
#include <string>
#include <vector>
+#include "arrow/status.h"
#include "arrow/type_fwd.h"
#include "arrow/util/macros.h"
-#include "arrow/util/status.h"
#include "arrow/util/visibility.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/CMakeLists.txt b/cpp/src/arrow/types/CMakeLists.txt
deleted file mode 100644
index 6d59acf..0000000
--- a/cpp/src/arrow/types/CMakeLists.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#######################################
-# arrow_types
-#######################################
-
-# Headers: top level
-install(FILES
- construct.h
- datetime.h
- decimal.h
- list.h
- primitive.h
- string.h
- struct.h
- union.h
- DESTINATION include/arrow/types)
-
-
-ADD_ARROW_TEST(decimal-test)
-ADD_ARROW_TEST(list-test)
-ADD_ARROW_TEST(primitive-test)
-ADD_ARROW_TEST(string-test)
-ADD_ARROW_TEST(struct-test)
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/construct.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/construct.cc b/cpp/src/arrow/types/construct.cc
deleted file mode 100644
index ab9c59f..0000000
--- a/cpp/src/arrow/types/construct.cc
+++ /dev/null
@@ -1,124 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/construct.h"
-
-#include <memory>
-
-#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-
-class ArrayBuilder;
-
-#define BUILDER_CASE(ENUM, BuilderType) \
- case Type::ENUM: \
- out->reset(new BuilderType(pool, type)); \
- return Status::OK();
-
-// Initially looked at doing this with vtables, but shared pointers makes it
-// difficult
-//
-// TODO(wesm): come up with a less monolithic strategy
-Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
- std::shared_ptr<ArrayBuilder>* out) {
- switch (type->type) {
- BUILDER_CASE(UINT8, UInt8Builder);
- BUILDER_CASE(INT8, Int8Builder);
- BUILDER_CASE(UINT16, UInt16Builder);
- BUILDER_CASE(INT16, Int16Builder);
- BUILDER_CASE(UINT32, UInt32Builder);
- BUILDER_CASE(INT32, Int32Builder);
- BUILDER_CASE(UINT64, UInt64Builder);
- BUILDER_CASE(INT64, Int64Builder);
- BUILDER_CASE(TIMESTAMP, TimestampBuilder);
-
- BUILDER_CASE(BOOL, BooleanBuilder);
-
- BUILDER_CASE(FLOAT, FloatBuilder);
- BUILDER_CASE(DOUBLE, DoubleBuilder);
-
- BUILDER_CASE(STRING, StringBuilder);
- BUILDER_CASE(BINARY, BinaryBuilder);
-
- case Type::LIST: {
- std::shared_ptr<ArrayBuilder> value_builder;
- std::shared_ptr<DataType> value_type =
- static_cast<ListType*>(type.get())->value_type();
- RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
- out->reset(new ListBuilder(pool, value_builder));
- return Status::OK();
- }
-
- case Type::STRUCT: {
- std::vector<FieldPtr>& fields = type->children_;
- std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
-
- for (auto it : fields) {
- std::shared_ptr<ArrayBuilder> builder;
- RETURN_NOT_OK(MakeBuilder(pool, it->type, &builder));
- values_builder.push_back(builder);
- }
- out->reset(new StructBuilder(pool, type, values_builder));
- return Status::OK();
- }
-
- default:
- return Status::NotImplemented(type->ToString());
- }
-}
-
-#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType) \
- case Type::ENUM: \
- out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
- break;
-
-Status MakePrimitiveArray(const TypePtr& type, int32_t length,
- const std::shared_ptr<Buffer>& data, int32_t null_count,
- const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
- switch (type->type) {
- MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
- MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
- MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
- MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
- MAKE_PRIMITIVE_ARRAY_CASE(INT16, Int16Array);
- MAKE_PRIMITIVE_ARRAY_CASE(UINT32, UInt32Array);
- MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
- MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
- MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
- MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
- MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
- MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
- MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
- MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
- default:
- return Status::NotImplemented(type->ToString());
- }
-#ifdef NDEBUG
- return Status::OK();
-#else
- return (*out)->Validate();
-#endif
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/construct.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h
deleted file mode 100644
index e18e946..0000000
--- a/cpp/src/arrow/types/construct.h
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_CONSTRUCT_H
-#define ARROW_TYPES_CONSTRUCT_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class ArrayBuilder;
-class Buffer;
-struct DataType;
-struct Field;
-class MemoryPool;
-class Status;
-
-Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
- std::shared_ptr<ArrayBuilder>* out);
-
-// Create new arrays for logical types that are backed by primitive arrays.
-Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
- int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
- const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
-
-} // namespace arrow
-
-#endif // ARROW_BUILDER_H_
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/datetime.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/datetime.h b/cpp/src/arrow/types/datetime.h
deleted file mode 100644
index a8f8639..0000000
--- a/cpp/src/arrow/types/datetime.h
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_DATETIME_H
-#define ARROW_TYPES_DATETIME_H
-
-#include <string>
-
-#include "arrow/type.h"
-
-namespace arrow {} // namespace arrow
-
-#endif // ARROW_TYPES_DATETIME_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/decimal-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/decimal-test.cc b/cpp/src/arrow/types/decimal-test.cc
deleted file mode 100644
index 7296ff8..0000000
--- a/cpp/src/arrow/types/decimal-test.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include "arrow/types/decimal.h"
-
-namespace arrow {
-
-TEST(TypesTest, TestDecimalType) {
- DecimalType t1(8, 4);
-
- ASSERT_EQ(t1.type, Type::DECIMAL);
- ASSERT_EQ(t1.precision, 8);
- ASSERT_EQ(t1.scale, 4);
-
- ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
-
- // Test copy constructor
- DecimalType t2 = t1;
- ASSERT_EQ(t2.type, Type::DECIMAL);
- ASSERT_EQ(t2.precision, 8);
- ASSERT_EQ(t2.scale, 4);
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/decimal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/decimal.cc b/cpp/src/arrow/types/decimal.cc
deleted file mode 100644
index 1d9a5e5..0000000
--- a/cpp/src/arrow/types/decimal.cc
+++ /dev/null
@@ -1,31 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/decimal.h"
-
-#include <sstream>
-#include <string>
-
-namespace arrow {
-
-std::string DecimalType::ToString() const {
- std::stringstream s;
- s << "decimal(" << precision << ", " << scale << ")";
- return s.str();
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/decimal.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h
deleted file mode 100644
index b3ea3a5..0000000
--- a/cpp/src/arrow/types/decimal.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_DECIMAL_H
-#define ARROW_TYPES_DECIMAL_H
-
-#include <string>
-
-#include "arrow/type.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {} // namespace arrow
-
-#endif // ARROW_TYPES_DECIMAL_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/list-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/list-test.cc b/cpp/src/arrow/types/list-test.cc
deleted file mode 100644
index cb9a8c1..0000000
--- a/cpp/src/arrow/types/list-test.cc
+++ /dev/null
@@ -1,241 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/test-common.h"
-#include "arrow/util/status.h"
-
-using std::shared_ptr;
-using std::string;
-using std::unique_ptr;
-using std::vector;
-
-namespace arrow {
-
-TEST(TypesTest, TestListType) {
- std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
-
- ListType list_type(vt);
- ASSERT_EQ(list_type.type, Type::LIST);
-
- ASSERT_EQ(list_type.name(), string("list"));
- ASSERT_EQ(list_type.ToString(), string("list<item: uint8>"));
-
- ASSERT_EQ(list_type.value_type()->type, vt->type);
- ASSERT_EQ(list_type.value_type()->type, vt->type);
-
- std::shared_ptr<DataType> st = std::make_shared<StringType>();
- std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
- ASSERT_EQ(lt->ToString(), string("list<item: string>"));
-
- ListType lt2(lt);
- ASSERT_EQ(lt2.ToString(), string("list<item: list<item: string>>"));
-}
-
-// ----------------------------------------------------------------------
-// List tests
-
-class TestListBuilder : public TestBuilder {
- public:
- void SetUp() {
- TestBuilder::SetUp();
-
- value_type_ = TypePtr(new Int32Type());
- type_ = TypePtr(new ListType(value_type_));
-
- std::shared_ptr<ArrayBuilder> tmp;
- ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
- builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
- }
-
- void Done() {
- std::shared_ptr<Array> out;
- EXPECT_OK(builder_->Finish(&out));
- result_ = std::dynamic_pointer_cast<ListArray>(out);
- }
-
- protected:
- TypePtr value_type_;
- TypePtr type_;
-
- shared_ptr<ListBuilder> builder_;
- shared_ptr<ListArray> result_;
-};
-
-TEST_F(TestListBuilder, Equality) {
- Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
-
- ArrayPtr array, equal_array, unequal_array;
- vector<int32_t> equal_offsets = {0, 1, 2, 5};
- vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2};
- vector<int32_t> unequal_offsets = {0, 1, 4};
- vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
-
- // setup two equal arrays
- ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
- ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
-
- ASSERT_OK(builder_->Finish(&array));
- ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
- ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
-
- ASSERT_OK(builder_->Finish(&equal_array));
- // now an unequal one
- ASSERT_OK(builder_->Append(unequal_offsets.data(), unequal_offsets.size()));
- ASSERT_OK(vb->Append(unequal_values.data(), unequal_values.size()));
-
- ASSERT_OK(builder_->Finish(&unequal_array));
-
- // Test array equality
- EXPECT_TRUE(array->Equals(array));
- EXPECT_TRUE(array->Equals(equal_array));
- EXPECT_TRUE(equal_array->Equals(array));
- EXPECT_FALSE(equal_array->Equals(unequal_array));
- EXPECT_FALSE(unequal_array->Equals(equal_array));
-
- // Test range equality
- EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_array));
- EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
- EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
- EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
- EXPECT_TRUE(array->RangeEquals(3, 4, 1, unequal_array));
-}
-
-TEST_F(TestListBuilder, TestResize) {}
-
-TEST_F(TestListBuilder, TestAppendNull) {
- ASSERT_OK(builder_->AppendNull());
- ASSERT_OK(builder_->AppendNull());
-
- Done();
-
- ASSERT_OK(result_->Validate());
- ASSERT_TRUE(result_->IsNull(0));
- ASSERT_TRUE(result_->IsNull(1));
-
- ASSERT_EQ(0, result_->raw_offsets()[0]);
- ASSERT_EQ(0, result_->offset(1));
- ASSERT_EQ(0, result_->offset(2));
-
- Int32Array* values = static_cast<Int32Array*>(result_->values().get());
- ASSERT_EQ(0, values->length());
-}
-
-void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
- const vector<uint8_t>& is_valid) {
- ASSERT_OK(result->Validate());
- ASSERT_EQ(1, result->null_count());
- ASSERT_EQ(0, result->values()->null_count());
-
- ASSERT_EQ(3, result->length());
- vector<int32_t> ex_offsets = {0, 3, 3, 7};
- for (size_t i = 0; i < ex_offsets.size(); ++i) {
- ASSERT_EQ(ex_offsets[i], result->offset(i));
- }
-
- for (int i = 0; i < result->length(); ++i) {
- ASSERT_EQ(!static_cast<bool>(is_valid[i]), result->IsNull(i));
- }
-
- ASSERT_EQ(7, result->values()->length());
- Int32Array* varr = static_cast<Int32Array*>(result->values().get());
-
- for (size_t i = 0; i < values.size(); ++i) {
- ASSERT_EQ(values[i], varr->Value(i));
- }
-}
-
-TEST_F(TestListBuilder, TestBasics) {
- vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
- vector<int> lengths = {3, 0, 4};
- vector<uint8_t> is_valid = {1, 0, 1};
-
- Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
-
- ASSERT_OK(builder_->Reserve(lengths.size()));
- ASSERT_OK(vb->Reserve(values.size()));
-
- int pos = 0;
- for (size_t i = 0; i < lengths.size(); ++i) {
- ASSERT_OK(builder_->Append(is_valid[i] > 0));
- for (int j = 0; j < lengths[i]; ++j) {
- vb->Append(values[pos++]);
- }
- }
-
- Done();
- ValidateBasicListArray(result_.get(), values, is_valid);
-}
-
-TEST_F(TestListBuilder, BulkAppend) {
- vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
- vector<int> lengths = {3, 0, 4};
- vector<uint8_t> is_valid = {1, 0, 1};
- vector<int32_t> offsets = {0, 3, 3};
-
- Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
- ASSERT_OK(vb->Reserve(values.size()));
-
- builder_->Append(offsets.data(), offsets.size(), is_valid.data());
- for (int32_t value : values) {
- vb->Append(value);
- }
- Done();
- ValidateBasicListArray(result_.get(), values, is_valid);
-}
-
-TEST_F(TestListBuilder, BulkAppendInvalid) {
- vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
- vector<int> lengths = {3, 0, 4};
- vector<uint8_t> is_null = {0, 1, 0};
- vector<uint8_t> is_valid = {1, 0, 1};
- vector<int32_t> offsets = {0, 2, 4}; // should be 0, 3, 3 given the is_null array
-
- Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
- ASSERT_OK(vb->Reserve(values.size()));
-
- builder_->Append(offsets.data(), offsets.size(), is_valid.data());
- builder_->Append(offsets.data(), offsets.size(), is_valid.data());
- for (int32_t value : values) {
- vb->Append(value);
- }
-
- Done();
- ASSERT_RAISES(Invalid, result_->Validate());
-}
-
-TEST_F(TestListBuilder, TestZeroLength) {
- // All buffers are null
- Done();
- ASSERT_OK(result_->Validate());
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/list.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/list.cc b/cpp/src/arrow/types/list.cc
deleted file mode 100644
index d865632..0000000
--- a/cpp/src/arrow/types/list.cc
+++ /dev/null
@@ -1,162 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-#include "arrow/types/list.h"
-
-#include <sstream>
-
-namespace arrow {
-
-bool ListArray::EqualsExact(const ListArray& other) const {
- if (this == &other) { return true; }
- if (null_count_ != other.null_count_) { return false; }
-
- bool equal_offsets =
- offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
- if (!equal_offsets) { return false; }
- bool equal_null_bitmap = true;
- if (null_count_ > 0) {
- equal_null_bitmap =
- null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
- }
-
- if (!equal_null_bitmap) { return false; }
-
- return values()->Equals(other.values());
-}
-
-bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- return EqualsExact(*static_cast<const ListArray*>(arr.get()));
-}
-
-bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- const auto other = static_cast<ListArray*>(arr.get());
- for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
- const bool is_null = IsNull(i);
- if (is_null != arr->IsNull(o_i)) { return false; }
- if (is_null) continue;
- const int32_t begin_offset = offset(i);
- const int32_t end_offset = offset(i + 1);
- const int32_t other_begin_offset = other->offset(o_i);
- const int32_t other_end_offset = other->offset(o_i + 1);
- // Underlying can't be equal if the size isn't equal
- if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
- return false;
- }
- if (!values_->RangeEquals(
- begin_offset, end_offset, other_begin_offset, other->values())) {
- return false;
- }
- }
- return true;
-}
-
-Status ListArray::Validate() const {
- if (length_ < 0) { return Status::Invalid("Length was negative"); }
- if (!offset_buffer_) { return Status::Invalid("offset_buffer_ was null"); }
- if (offset_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
- std::stringstream ss;
- ss << "offset buffer size (bytes): " << offset_buffer_->size()
- << " isn't large enough for length: " << length_;
- return Status::Invalid(ss.str());
- }
- const int32_t last_offset = offset(length_);
- if (last_offset > 0) {
- if (!values_) {
- return Status::Invalid("last offset was non-zero and values was null");
- }
- if (values_->length() != last_offset) {
- std::stringstream ss;
- ss << "Final offset invariant not equal to values length: " << last_offset
- << "!=" << values_->length();
- return Status::Invalid(ss.str());
- }
-
- const Status child_valid = values_->Validate();
- if (!child_valid.ok()) {
- std::stringstream ss;
- ss << "Child array invalid: " << child_valid.ToString();
- return Status::Invalid(ss.str());
- }
- }
-
- int32_t prev_offset = offset(0);
- if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
- for (int32_t i = 1; i <= length_; ++i) {
- int32_t current_offset = offset(i);
- if (IsNull(i - 1) && current_offset != prev_offset) {
- std::stringstream ss;
- ss << "Offset invariant failure at: " << i << " inconsistent offsets for null slot"
- << current_offset << "!=" << prev_offset;
- return Status::Invalid(ss.str());
- }
- if (current_offset < prev_offset) {
- std::stringstream ss;
- ss << "Offset invariant failure: " << i
- << " inconsistent offset for non-null slot: " << current_offset << "<"
- << prev_offset;
- return Status::Invalid(ss.str());
- }
- prev_offset = current_offset;
- }
- return Status::OK();
-}
-
-Status ListBuilder::Init(int32_t elements) {
- DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
- RETURN_NOT_OK(ArrayBuilder::Init(elements));
- // one more then requested for offsets
- return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
-}
-
-Status ListBuilder::Resize(int32_t capacity) {
- DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
- // one more then requested for offsets
- RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
- return ArrayBuilder::Resize(capacity);
-}
-
-Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
- std::shared_ptr<Array> items = values_;
- if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
-
- RETURN_NOT_OK(offset_builder_.Append<int32_t>(items->length()));
- std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
-
- *out = std::make_shared<ListArray>(
- type_, length_, offsets, items, null_count_, null_bitmap_);
-
- Reset();
-
- return Status::OK();
-}
-
-void ListBuilder::Reset() {
- capacity_ = length_ = null_count_ = 0;
- null_bitmap_ = nullptr;
-}
-
-Status ListArray::Accept(ArrayVisitor* visitor) const {
- return visitor->Visit(*this);
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/list.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h
deleted file mode 100644
index ec09a78..0000000
--- a/cpp/src/arrow/types/list.h
+++ /dev/null
@@ -1,170 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_LIST_H
-#define ARROW_TYPES_LIST_H
-
-#include <cstdint>
-#include <cstring>
-#include <limits>
-#include <memory>
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/logging.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class MemoryPool;
-
-class ARROW_EXPORT ListArray : public Array {
- public:
- using TypeClass = ListType;
-
- ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
- const ArrayPtr& values, int32_t null_count = 0,
- std::shared_ptr<Buffer> null_bitmap = nullptr)
- : Array(type, length, null_count, null_bitmap) {
- offset_buffer_ = offsets;
- offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
- offset_buffer_->data());
- values_ = values;
- }
-
- Status Validate() const override;
-
- virtual ~ListArray() = default;
-
- // Return a shared pointer in case the requestor desires to share ownership
- // with this array.
- std::shared_ptr<Array> values() const { return values_; }
- std::shared_ptr<Buffer> offsets() const {
- return std::static_pointer_cast<Buffer>(offset_buffer_);
- }
-
- std::shared_ptr<DataType> value_type() const { return values_->type(); }
-
- const int32_t* raw_offsets() const { return offsets_; }
-
- int32_t offset(int i) const { return offsets_[i]; }
-
- // Neither of these functions will perform boundschecking
- int32_t value_offset(int i) const { return offsets_[i]; }
- int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
-
- bool EqualsExact(const ListArray& other) const;
- bool Equals(const std::shared_ptr<Array>& arr) const override;
-
- bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const ArrayPtr& arr) const override;
-
- Status Accept(ArrayVisitor* visitor) const override;
-
- protected:
- std::shared_ptr<Buffer> offset_buffer_;
- const int32_t* offsets_;
- ArrayPtr values_;
-};
-
-// ----------------------------------------------------------------------
-// Array builder
-
-// Builder class for variable-length list array value types
-//
-// To use this class, you must append values to the child array builder and use
-// the Append function to delimit each distinct list value (once the values
-// have been appended to the child array) or use the bulk API to append
-// a sequence of offests and null values.
-//
-// A note on types. Per arrow/type.h all types in the c++ implementation are
-// logical so even though this class always builds list array, this can
-// represent multiple different logical types. If no logical type is provided
-// at construction time, the class defaults to List<T> where t is taken from the
-// value_builder/values that the object is constructed with.
-class ARROW_EXPORT ListBuilder : public ArrayBuilder {
- public:
- // Use this constructor to incrementally build the value array along with offsets and
- // null bitmap.
- ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
- const TypePtr& type = nullptr)
- : ArrayBuilder(
- pool, type ? type : std::static_pointer_cast<DataType>(
- std::make_shared<ListType>(value_builder->type()))),
- offset_builder_(pool),
- value_builder_(value_builder) {}
-
- // Use this constructor to build the list with a pre-existing values array
- ListBuilder(
- MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr)
- : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
- std::make_shared<ListType>(values->type()))),
- offset_builder_(pool),
- values_(values) {}
-
- virtual ~ListBuilder() {}
-
- Status Init(int32_t elements) override;
- Status Resize(int32_t capacity) override;
- Status Finish(std::shared_ptr<Array>* out) override;
-
- // Vector append
- //
- // If passed, valid_bytes is of equal length to values, and any zero byte
- // will be considered as a null for that slot
- Status Append(
- const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
- RETURN_NOT_OK(Reserve(length));
- UnsafeAppendToBitmap(valid_bytes, length);
- offset_builder_.UnsafeAppend<int32_t>(offsets, length);
- return Status::OK();
- }
-
- // Start a new variable-length list slot
- //
- // This function should be called before beginning to append elements to the
- // value builder
- Status Append(bool is_valid = true) {
- RETURN_NOT_OK(Reserve(1));
- UnsafeAppendToBitmap(is_valid);
- RETURN_NOT_OK(offset_builder_.Append<int32_t>(value_builder_->length()));
- return Status::OK();
- }
-
- Status AppendNull() { return Append(false); }
-
- std::shared_ptr<ArrayBuilder> value_builder() const {
- DCHECK(!values_) << "Using value builder is pointless when values_ is set";
- return value_builder_;
- }
-
- protected:
- BufferBuilder offset_builder_;
- std::shared_ptr<ArrayBuilder> value_builder_;
- std::shared_ptr<Array> values_;
-
- void Reset();
-};
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_LIST_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/primitive-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/primitive-test.cc b/cpp/src/arrow/types/primitive-test.cc
deleted file mode 100644
index bdc8ec0..0000000
--- a/cpp/src/arrow/types/primitive-test.cc
+++ /dev/null
@@ -1,478 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/type_traits.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/test-common.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-
-using std::string;
-using std::shared_ptr;
-using std::unique_ptr;
-using std::vector;
-
-namespace arrow {
-
-class Array;
-
-#define PRIMITIVE_TEST(KLASS, ENUM, NAME) \
- TEST(TypesTest, TestPrimitive_##ENUM) { \
- KLASS tp; \
- \
- ASSERT_EQ(tp.type, Type::ENUM); \
- ASSERT_EQ(tp.ToString(), string(NAME)); \
- \
- KLASS tp_copy = tp; \
- ASSERT_EQ(tp_copy.type, Type::ENUM); \
- }
-
-PRIMITIVE_TEST(Int8Type, INT8, "int8");
-PRIMITIVE_TEST(Int16Type, INT16, "int16");
-PRIMITIVE_TEST(Int32Type, INT32, "int32");
-PRIMITIVE_TEST(Int64Type, INT64, "int64");
-PRIMITIVE_TEST(UInt8Type, UINT8, "uint8");
-PRIMITIVE_TEST(UInt16Type, UINT16, "uint16");
-PRIMITIVE_TEST(UInt32Type, UINT32, "uint32");
-PRIMITIVE_TEST(UInt64Type, UINT64, "uint64");
-
-PRIMITIVE_TEST(FloatType, FLOAT, "float");
-PRIMITIVE_TEST(DoubleType, DOUBLE, "double");
-
-PRIMITIVE_TEST(BooleanType, BOOL, "bool");
-
-// ----------------------------------------------------------------------
-// Primitive type tests
-
-TEST_F(TestBuilder, TestReserve) {
- builder_->Init(10);
- ASSERT_EQ(2, builder_->null_bitmap()->size());
-
- builder_->Reserve(30);
- ASSERT_EQ(4, builder_->null_bitmap()->size());
-}
-
-template <typename Attrs>
-class TestPrimitiveBuilder : public TestBuilder {
- public:
- typedef typename Attrs::ArrayType ArrayType;
- typedef typename Attrs::BuilderType BuilderType;
- typedef typename Attrs::T T;
- typedef typename Attrs::Type Type;
-
- virtual void SetUp() {
- TestBuilder::SetUp();
-
- type_ = Attrs::type();
-
- std::shared_ptr<ArrayBuilder> tmp;
- ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
- builder_ = std::dynamic_pointer_cast<BuilderType>(tmp);
-
- ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
- builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
- }
-
- void RandomData(int N, double pct_null = 0.1) {
- Attrs::draw(N, &draws_);
-
- valid_bytes_.resize(N);
- test::random_null_bytes(N, pct_null, valid_bytes_.data());
- }
-
- void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
- int size = builder->length();
-
- auto ex_data = std::make_shared<Buffer>(
- reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
-
- std::shared_ptr<Buffer> ex_null_bitmap;
- int32_t ex_null_count = 0;
-
- if (nullable) {
- ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
- ex_null_count = test::null_count(valid_bytes_);
- } else {
- ex_null_bitmap = nullptr;
- }
-
- auto expected =
- std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_null_bitmap);
-
- std::shared_ptr<Array> out;
- ASSERT_OK(builder->Finish(&out));
-
- std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(out);
-
- // Builder is now reset
- ASSERT_EQ(0, builder->length());
- ASSERT_EQ(0, builder->capacity());
- ASSERT_EQ(0, builder->null_count());
- ASSERT_EQ(nullptr, builder->data());
-
- ASSERT_EQ(ex_null_count, result->null_count());
- ASSERT_TRUE(result->EqualsExact(*expected.get()));
- }
-
- protected:
- std::shared_ptr<DataType> type_;
- shared_ptr<BuilderType> builder_;
- shared_ptr<BuilderType> builder_nn_;
-
- vector<T> draws_;
- vector<uint8_t> valid_bytes_;
-};
-
-#define PTYPE_DECL(CapType, c_type) \
- typedef CapType##Array ArrayType; \
- typedef CapType##Builder BuilderType; \
- typedef CapType##Type Type; \
- typedef c_type T; \
- \
- static std::shared_ptr<DataType> type() { \
- return std::shared_ptr<DataType>(new Type()); \
- }
-
-#define PINT_DECL(CapType, c_type, LOWER, UPPER) \
- struct P##CapType { \
- PTYPE_DECL(CapType, c_type); \
- static void draw(int N, vector<T>* draws) { \
- test::randint<T>(N, LOWER, UPPER, draws); \
- } \
- }
-
-#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER) \
- struct P##CapType { \
- PTYPE_DECL(CapType, c_type); \
- static void draw(int N, vector<T>* draws) { \
- test::random_real<T>(N, 0, LOWER, UPPER, draws); \
- } \
- }
-
-PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
-PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
-PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
-PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX);
-
-PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX);
-PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
-PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
-PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
-
-PFLOAT_DECL(Float, float, -1000, 1000);
-PFLOAT_DECL(Double, double, -1000, 1000);
-
-struct PBoolean {
- PTYPE_DECL(Boolean, uint8_t);
-};
-
-template <>
-void TestPrimitiveBuilder<PBoolean>::RandomData(int N, double pct_null) {
- draws_.resize(N);
- valid_bytes_.resize(N);
-
- test::random_null_bytes(N, 0.5, draws_.data());
- test::random_null_bytes(N, pct_null, valid_bytes_.data());
-}
-
-template <>
-void TestPrimitiveBuilder<PBoolean>::Check(
- const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
- int size = builder->length();
-
- auto ex_data = test::bytes_to_null_buffer(draws_);
-
- std::shared_ptr<Buffer> ex_null_bitmap;
- int32_t ex_null_count = 0;
-
- if (nullable) {
- ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
- ex_null_count = test::null_count(valid_bytes_);
- } else {
- ex_null_bitmap = nullptr;
- }
-
- auto expected =
- std::make_shared<BooleanArray>(size, ex_data, ex_null_count, ex_null_bitmap);
-
- std::shared_ptr<Array> out;
- ASSERT_OK(builder->Finish(&out));
- std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(out);
-
- // Builder is now reset
- ASSERT_EQ(0, builder->length());
- ASSERT_EQ(0, builder->capacity());
- ASSERT_EQ(0, builder->null_count());
- ASSERT_EQ(nullptr, builder->data());
-
- ASSERT_EQ(ex_null_count, result->null_count());
-
- ASSERT_EQ(expected->length(), result->length());
-
- for (int i = 0; i < result->length(); ++i) {
- if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
- bool actual = BitUtil::GetBit(result->raw_data(), i);
- ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
- }
- ASSERT_TRUE(result->EqualsExact(*expected.get()));
-}
-
-typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
- PInt32, PInt64, PFloat, PDouble>
- Primitives;
-
-TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
-
-#define DECL_T() typedef typename TestFixture::T T;
-
-#define DECL_TYPE() typedef typename TestFixture::Type Type;
-
-#define DECL_ARRAYTYPE() typedef typename TestFixture::ArrayType ArrayType;
-
-TYPED_TEST(TestPrimitiveBuilder, TestInit) {
- DECL_TYPE();
-
- int n = 1000;
- ASSERT_OK(this->builder_->Reserve(n));
- ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
- ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
- this->builder_->data()->size());
-
- // unsure if this should go in all builder classes
- ASSERT_EQ(0, this->builder_->num_children());
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
- int size = 1000;
- for (int i = 0; i < size; ++i) {
- ASSERT_OK(this->builder_->AppendNull());
- }
-
- std::shared_ptr<Array> result;
- ASSERT_OK(this->builder_->Finish(&result));
-
- for (int i = 0; i < size; ++i) {
- ASSERT_TRUE(result->IsNull(i)) << i;
- }
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
- DECL_T();
-
- int size = 1000;
-
- vector<T>& draws = this->draws_;
- vector<uint8_t>& valid_bytes = this->valid_bytes_;
-
- int64_t memory_before = this->pool_->bytes_allocated();
-
- this->RandomData(size);
-
- this->builder_->Reserve(size);
-
- int i;
- for (i = 0; i < size; ++i) {
- if (valid_bytes[i] > 0) {
- this->builder_->Append(draws[i]);
- } else {
- this->builder_->AppendNull();
- }
- }
-
- do {
- std::shared_ptr<Array> result;
- ASSERT_OK(this->builder_->Finish(&result));
- } while (false);
-
- ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
-}
-
-TYPED_TEST(TestPrimitiveBuilder, Equality) {
- DECL_T();
-
- const int size = 1000;
- this->RandomData(size);
- vector<T>& draws = this->draws_;
- vector<uint8_t>& valid_bytes = this->valid_bytes_;
- ArrayPtr array, equal_array, unequal_array;
- auto builder = this->builder_.get();
- ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
- ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
-
- // Make the not equal array by negating the first valid element with itself.
- const auto first_valid = std::find_if(
- valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
- const int first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
- // This should be true with a very high probability, but might introduce flakiness
- ASSERT_LT(first_valid_idx, size - 1);
- draws[first_valid_idx] = ~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]);
- ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
-
- // test normal equality
- EXPECT_TRUE(array->Equals(array));
- EXPECT_TRUE(array->Equals(equal_array));
- EXPECT_TRUE(equal_array->Equals(array));
- EXPECT_FALSE(equal_array->Equals(unequal_array));
- EXPECT_FALSE(unequal_array->Equals(equal_array));
-
- // Test range equality
- EXPECT_FALSE(array->RangeEquals(0, first_valid_idx + 1, 0, unequal_array));
- EXPECT_FALSE(array->RangeEquals(first_valid_idx, size, first_valid_idx, unequal_array));
- EXPECT_TRUE(array->RangeEquals(0, first_valid_idx, 0, unequal_array));
- EXPECT_TRUE(
- array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
- DECL_T();
-
- const int size = 10000;
-
- vector<T>& draws = this->draws_;
- vector<uint8_t>& valid_bytes = this->valid_bytes_;
-
- this->RandomData(size);
-
- this->builder_->Reserve(1000);
- this->builder_nn_->Reserve(1000);
-
- int i;
- int null_count = 0;
- // Append the first 1000
- for (i = 0; i < 1000; ++i) {
- if (valid_bytes[i] > 0) {
- this->builder_->Append(draws[i]);
- } else {
- this->builder_->AppendNull();
- ++null_count;
- }
- this->builder_nn_->Append(draws[i]);
- }
-
- ASSERT_EQ(null_count, this->builder_->null_count());
-
- ASSERT_EQ(1000, this->builder_->length());
- ASSERT_EQ(1024, this->builder_->capacity());
-
- ASSERT_EQ(1000, this->builder_nn_->length());
- ASSERT_EQ(1024, this->builder_nn_->capacity());
-
- this->builder_->Reserve(size - 1000);
- this->builder_nn_->Reserve(size - 1000);
-
- // Append the next 9000
- for (i = 1000; i < size; ++i) {
- if (valid_bytes[i] > 0) {
- this->builder_->Append(draws[i]);
- } else {
- this->builder_->AppendNull();
- }
- this->builder_nn_->Append(draws[i]);
- }
-
- ASSERT_EQ(size, this->builder_->length());
- ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
-
- ASSERT_EQ(size, this->builder_nn_->length());
- ASSERT_EQ(BitUtil::NextPower2(size), this->builder_nn_->capacity());
-
- this->Check(this->builder_, true);
- this->Check(this->builder_nn_, false);
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
- DECL_T();
-
- int size = 10000;
- this->RandomData(size);
-
- vector<T>& draws = this->draws_;
- vector<uint8_t>& valid_bytes = this->valid_bytes_;
-
- // first slug
- int K = 1000;
-
- ASSERT_OK(this->builder_->Append(draws.data(), K, valid_bytes.data()));
- ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
-
- ASSERT_EQ(1000, this->builder_->length());
- ASSERT_EQ(1024, this->builder_->capacity());
-
- ASSERT_EQ(1000, this->builder_nn_->length());
- ASSERT_EQ(1024, this->builder_nn_->capacity());
-
- // Append the next 9000
- ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, valid_bytes.data() + K));
- ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
-
- ASSERT_EQ(size, this->builder_->length());
- ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
-
- this->Check(this->builder_, true);
- this->Check(this->builder_nn_, false);
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
- int n = 1000;
- ASSERT_OK(this->builder_->Reserve(n));
-
- ASSERT_OK(this->builder_->Advance(100));
- ASSERT_EQ(100, this->builder_->length());
-
- ASSERT_OK(this->builder_->Advance(900));
-
- int too_many = this->builder_->capacity() - 1000 + 1;
- ASSERT_RAISES(Invalid, this->builder_->Advance(too_many));
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestResize) {
- DECL_TYPE();
-
- int cap = kMinBuilderCapacity * 2;
-
- ASSERT_OK(this->builder_->Reserve(cap));
- ASSERT_EQ(cap, this->builder_->capacity());
-
- ASSERT_EQ(TypeTraits<Type>::bytes_required(cap), this->builder_->data()->size());
- ASSERT_EQ(BitUtil::BytesForBits(cap), this->builder_->null_bitmap()->size());
-}
-
-TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
- ASSERT_OK(this->builder_->Reserve(10));
- ASSERT_EQ(0, this->builder_->length());
- ASSERT_EQ(kMinBuilderCapacity, this->builder_->capacity());
-
- ASSERT_OK(this->builder_->Reserve(90));
- ASSERT_OK(this->builder_->Advance(100));
- ASSERT_OK(this->builder_->Reserve(kMinBuilderCapacity));
-
- ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/primitive.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/primitive.cc b/cpp/src/arrow/types/primitive.cc
deleted file mode 100644
index 75e5a9f..0000000
--- a/cpp/src/arrow/types/primitive.cc
+++ /dev/null
@@ -1,294 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/primitive.h"
-
-#include <cmath>
-#include <memory>
-
-#include "arrow/type_traits.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/logging.h"
-
-namespace arrow {
-
-// ----------------------------------------------------------------------
-// Primitive array base
-
-PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
- const std::shared_ptr<Buffer>& data, int32_t null_count,
- const std::shared_ptr<Buffer>& null_bitmap)
- : Array(type, length, null_count, null_bitmap) {
- data_ = data;
- raw_data_ = data == nullptr ? nullptr : data_->data();
-}
-
-bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
- if (this == &other) { return true; }
- if (null_count_ != other.null_count_) { return false; }
-
- if (null_count_ > 0) {
- bool equal_bitmap =
- null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
- if (!equal_bitmap) { return false; }
-
- const uint8_t* this_data = raw_data_;
- const uint8_t* other_data = other.raw_data_;
-
- auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
- int value_byte_size = size_meta->bit_width() / 8;
- DCHECK_GT(value_byte_size, 0);
-
- for (int i = 0; i < length_; ++i) {
- if (!IsNull(i) && memcmp(this_data, other_data, value_byte_size)) { return false; }
- this_data += value_byte_size;
- other_data += value_byte_size;
- }
- return true;
- } else {
- if (length_ == 0 && other.length_ == 0) { return true; }
- return data_->Equals(*other.data_, length_);
- }
-}
-
-bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
-}
-
-template <typename T>
-Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
- return visitor->Visit(*this);
-}
-
-template class NumericArray<UInt8Type>;
-template class NumericArray<UInt16Type>;
-template class NumericArray<UInt32Type>;
-template class NumericArray<UInt64Type>;
-template class NumericArray<Int8Type>;
-template class NumericArray<Int16Type>;
-template class NumericArray<Int32Type>;
-template class NumericArray<Int64Type>;
-template class NumericArray<TimestampType>;
-template class NumericArray<HalfFloatType>;
-template class NumericArray<FloatType>;
-template class NumericArray<DoubleType>;
-
-template <typename T>
-Status PrimitiveBuilder<T>::Init(int32_t capacity) {
- RETURN_NOT_OK(ArrayBuilder::Init(capacity));
- data_ = std::make_shared<PoolBuffer>(pool_);
-
- int64_t nbytes = TypeTraits<T>::bytes_required(capacity);
- RETURN_NOT_OK(data_->Resize(nbytes));
- // TODO(emkornfield) valgrind complains without this
- memset(data_->mutable_data(), 0, nbytes);
-
- raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
- return Status::OK();
-}
-
-template <typename T>
-Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
- // XXX: Set floor size for now
- if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
-
- if (capacity_ == 0) {
- RETURN_NOT_OK(Init(capacity));
- } else {
- RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
- const int64_t old_bytes = data_->size();
- const int64_t new_bytes = TypeTraits<T>::bytes_required(capacity);
- RETURN_NOT_OK(data_->Resize(new_bytes));
- raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
- memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
- }
- return Status::OK();
-}
-
-template <typename T>
-Status PrimitiveBuilder<T>::Append(
- const value_type* values, int32_t length, const uint8_t* valid_bytes) {
- RETURN_NOT_OK(Reserve(length));
-
- if (length > 0) {
- memcpy(raw_data_ + length_, values, TypeTraits<T>::bytes_required(length));
- }
-
- // length_ is update by these
- ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
-
- return Status::OK();
-}
-
-template <typename T>
-Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
- const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
- if (bytes_required > 0 && bytes_required < data_->size()) {
- // Trim buffers
- RETURN_NOT_OK(data_->Resize(bytes_required));
- }
- *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
- type_, length_, data_, null_count_, null_bitmap_);
-
- data_ = null_bitmap_ = nullptr;
- capacity_ = length_ = null_count_ = 0;
- return Status::OK();
-}
-
-template class PrimitiveBuilder<UInt8Type>;
-template class PrimitiveBuilder<UInt16Type>;
-template class PrimitiveBuilder<UInt32Type>;
-template class PrimitiveBuilder<UInt64Type>;
-template class PrimitiveBuilder<Int8Type>;
-template class PrimitiveBuilder<Int16Type>;
-template class PrimitiveBuilder<Int32Type>;
-template class PrimitiveBuilder<Int64Type>;
-template class PrimitiveBuilder<TimestampType>;
-template class PrimitiveBuilder<HalfFloatType>;
-template class PrimitiveBuilder<FloatType>;
-template class PrimitiveBuilder<DoubleType>;
-
-Status BooleanBuilder::Init(int32_t capacity) {
- RETURN_NOT_OK(ArrayBuilder::Init(capacity));
- data_ = std::make_shared<PoolBuffer>(pool_);
-
- int64_t nbytes = BitUtil::BytesForBits(capacity);
- RETURN_NOT_OK(data_->Resize(nbytes));
- // TODO(emkornfield) valgrind complains without this
- memset(data_->mutable_data(), 0, nbytes);
-
- raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
- return Status::OK();
-}
-
-Status BooleanBuilder::Resize(int32_t capacity) {
- // XXX: Set floor size for now
- if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
-
- if (capacity_ == 0) {
- RETURN_NOT_OK(Init(capacity));
- } else {
- RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
- const int64_t old_bytes = data_->size();
- const int64_t new_bytes = BitUtil::BytesForBits(capacity);
-
- RETURN_NOT_OK(data_->Resize(new_bytes));
- raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
- memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
- }
- return Status::OK();
-}
-
-Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
- const int64_t bytes_required = BitUtil::BytesForBits(length_);
-
- if (bytes_required > 0 && bytes_required < data_->size()) {
- // Trim buffers
- RETURN_NOT_OK(data_->Resize(bytes_required));
- }
- *out = std::make_shared<BooleanArray>(type_, length_, data_, null_count_, null_bitmap_);
-
- data_ = null_bitmap_ = nullptr;
- capacity_ = length_ = null_count_ = 0;
- return Status::OK();
-}
-
-Status BooleanBuilder::Append(
- const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
- RETURN_NOT_OK(Reserve(length));
-
- for (int i = 0; i < length; ++i) {
- // Skip reading from unitialised memory
- // TODO: This actually is only to keep valgrind happy but may or may not
- // have a performance impact.
- if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
-
- if (values[i] > 0) {
- BitUtil::SetBit(raw_data_, length_ + i);
- } else {
- BitUtil::ClearBit(raw_data_, length_ + i);
- }
- }
-
- // this updates length_
- ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
- return Status::OK();
-}
-
-BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
- int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
- : PrimitiveArray(
- std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
-
-BooleanArray::BooleanArray(const TypePtr& type, int32_t length,
- const std::shared_ptr<Buffer>& data, int32_t null_count,
- const std::shared_ptr<Buffer>& null_bitmap)
- : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
-
-bool BooleanArray::EqualsExact(const BooleanArray& other) const {
- if (this == &other) return true;
- if (null_count_ != other.null_count_) { return false; }
-
- if (null_count_ > 0) {
- bool equal_bitmap =
- null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
- if (!equal_bitmap) { return false; }
-
- const uint8_t* this_data = raw_data_;
- const uint8_t* other_data = other.raw_data_;
-
- for (int i = 0; i < length_; ++i) {
- if (!IsNull(i) && BitUtil::GetBit(this_data, i) != BitUtil::GetBit(other_data, i)) {
- return false;
- }
- }
- return true;
- } else {
- return data_->Equals(*other.data_, BitUtil::BytesForBits(length_));
- }
-}
-
-bool BooleanArray::Equals(const ArrayPtr& arr) const {
- if (this == arr.get()) return true;
- if (Type::BOOL != arr->type_enum()) { return false; }
- return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
-}
-
-bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
- int32_t other_start_idx, const ArrayPtr& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- const auto other = static_cast<BooleanArray*>(arr.get());
- for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
- const bool is_null = IsNull(i);
- if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
- return false;
- }
- }
- return true;
-}
-
-Status BooleanArray::Accept(ArrayVisitor* visitor) const {
- return visitor->Visit(*this);
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/primitive.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h
deleted file mode 100644
index ec578e1..0000000
--- a/cpp/src/arrow/types/primitive.h
+++ /dev/null
@@ -1,371 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_PRIMITIVE_H
-#define ARROW_TYPES_PRIMITIVE_H
-
-#include <cmath>
-#include <cstdint>
-#include <cstring>
-#include <iostream>
-#include <memory>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/type.h"
-#include "arrow/type_fwd.h"
-#include "arrow/types/datetime.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class MemoryPool;
-
-// Base class for fixed-size logical types. See MakePrimitiveArray
-// (types/construct.h) for constructing a specific subclass.
-class ARROW_EXPORT PrimitiveArray : public Array {
- public:
- virtual ~PrimitiveArray() {}
-
- std::shared_ptr<Buffer> data() const { return data_; }
-
- bool EqualsExact(const PrimitiveArray& other) const;
- bool Equals(const std::shared_ptr<Array>& arr) const override;
-
- protected:
- PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
- int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
- std::shared_ptr<Buffer> data_;
- const uint8_t* raw_data_;
-};
-
-template <typename TYPE>
-class ARROW_EXPORT NumericArray : public PrimitiveArray {
- public:
- using TypeClass = TYPE;
- using value_type = typename TypeClass::c_type;
- NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
- int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
- : PrimitiveArray(
- std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
- NumericArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
- int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
- : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
-
- bool EqualsExact(const NumericArray<TypeClass>& other) const {
- return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
- }
-
- bool ApproxEquals(const std::shared_ptr<Array>& arr) const { return Equals(arr); }
-
- bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const ArrayPtr& arr) const override {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- const auto other = static_cast<NumericArray<TypeClass>*>(arr.get());
- for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
- const bool is_null = IsNull(i);
- if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
- return false;
- }
- }
- return true;
- }
- const value_type* raw_data() const {
- return reinterpret_cast<const value_type*>(raw_data_);
- }
-
- Status Accept(ArrayVisitor* visitor) const override;
-
- value_type Value(int i) const { return raw_data()[i]; }
-};
-
-template <>
-inline bool NumericArray<FloatType>::ApproxEquals(
- const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
-
- const auto& other = *static_cast<NumericArray<FloatType>*>(arr.get());
-
- if (this == &other) { return true; }
- if (null_count_ != other.null_count_) { return false; }
-
- auto this_data = reinterpret_cast<const float*>(raw_data_);
- auto other_data = reinterpret_cast<const float*>(other.raw_data_);
-
- static constexpr float EPSILON = 1E-5;
-
- if (length_ == 0 && other.length_ == 0) { return true; }
-
- if (null_count_ > 0) {
- bool equal_bitmap =
- null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
- if (!equal_bitmap) { return false; }
-
- for (int i = 0; i < length_; ++i) {
- if (IsNull(i)) continue;
- if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
- }
- } else {
- for (int i = 0; i < length_; ++i) {
- if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
- }
- }
- return true;
-}
-
-template <>
-inline bool NumericArray<DoubleType>::ApproxEquals(
- const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
-
- const auto& other = *static_cast<NumericArray<DoubleType>*>(arr.get());
-
- if (this == &other) { return true; }
- if (null_count_ != other.null_count_) { return false; }
-
- auto this_data = reinterpret_cast<const double*>(raw_data_);
- auto other_data = reinterpret_cast<const double*>(other.raw_data_);
-
- if (length_ == 0 && other.length_ == 0) { return true; }
-
- static constexpr double EPSILON = 1E-5;
-
- if (null_count_ > 0) {
- bool equal_bitmap =
- null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
- if (!equal_bitmap) { return false; }
-
- for (int i = 0; i < length_; ++i) {
- if (IsNull(i)) continue;
- if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
- }
- } else {
- for (int i = 0; i < length_; ++i) {
- if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
- }
- }
- return true;
-}
-
-template <typename Type>
-class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
- public:
- using value_type = typename Type::c_type;
-
- explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
- : ArrayBuilder(pool, type), data_(nullptr) {}
-
- virtual ~PrimitiveBuilder() {}
-
- using ArrayBuilder::Advance;
-
- // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
- Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
- RETURN_NOT_OK(Reserve(length));
- UnsafeAppendToBitmap(valid_bytes, length);
- return Status::OK();
- }
-
- Status AppendNull() {
- RETURN_NOT_OK(Reserve(1));
- UnsafeAppendToBitmap(false);
- return Status::OK();
- }
-
- std::shared_ptr<Buffer> data() const { return data_; }
-
- // Vector append
- //
- // If passed, valid_bytes is of equal length to values, and any zero byte
- // will be considered as a null for that slot
- Status Append(
- const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
-
- Status Finish(std::shared_ptr<Array>* out) override;
- Status Init(int32_t capacity) override;
-
- // Increase the capacity of the builder to accommodate at least the indicated
- // number of elements
- Status Resize(int32_t capacity) override;
-
- protected:
- std::shared_ptr<PoolBuffer> data_;
- value_type* raw_data_;
-};
-
-template <typename T>
-class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
- public:
- using typename PrimitiveBuilder<T>::value_type;
- using PrimitiveBuilder<T>::PrimitiveBuilder;
-
- using PrimitiveBuilder<T>::Append;
- using PrimitiveBuilder<T>::Init;
- using PrimitiveBuilder<T>::Resize;
- using PrimitiveBuilder<T>::Reserve;
-
- // Scalar append.
- Status Append(value_type val) {
- RETURN_NOT_OK(ArrayBuilder::Reserve(1));
- UnsafeAppend(val);
- return Status::OK();
- }
-
- // Does not capacity-check; make sure to call Reserve beforehand
- void UnsafeAppend(value_type val) {
- BitUtil::SetBit(null_bitmap_data_, length_);
- raw_data_[length_++] = val;
- }
-
- protected:
- using PrimitiveBuilder<T>::length_;
- using PrimitiveBuilder<T>::null_bitmap_data_;
- using PrimitiveBuilder<T>::raw_data_;
-};
-
-// Builders
-
-using UInt8Builder = NumericBuilder<UInt8Type>;
-using UInt16Builder = NumericBuilder<UInt16Type>;
-using UInt32Builder = NumericBuilder<UInt32Type>;
-using UInt64Builder = NumericBuilder<UInt64Type>;
-
-using Int8Builder = NumericBuilder<Int8Type>;
-using Int16Builder = NumericBuilder<Int16Type>;
-using Int32Builder = NumericBuilder<Int32Type>;
-using Int64Builder = NumericBuilder<Int64Type>;
-using TimestampBuilder = NumericBuilder<TimestampType>;
-
-using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
-using FloatBuilder = NumericBuilder<FloatType>;
-using DoubleBuilder = NumericBuilder<DoubleType>;
-
-class ARROW_EXPORT BooleanArray : public PrimitiveArray {
- public:
- using TypeClass = BooleanType;
-
- BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
- int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
- BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
- int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
- bool EqualsExact(const BooleanArray& other) const;
- bool Equals(const ArrayPtr& arr) const override;
- bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const ArrayPtr& arr) const override;
-
- Status Accept(ArrayVisitor* visitor) const override;
-
- const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
-
- bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
-};
-
-class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
- public:
- explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
- : ArrayBuilder(pool, type), data_(nullptr) {}
-
- virtual ~BooleanBuilder() {}
-
- using ArrayBuilder::Advance;
-
- // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
- Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
- RETURN_NOT_OK(Reserve(length));
- UnsafeAppendToBitmap(valid_bytes, length);
- return Status::OK();
- }
-
- Status AppendNull() {
- RETURN_NOT_OK(Reserve(1));
- UnsafeAppendToBitmap(false);
- return Status::OK();
- }
-
- std::shared_ptr<Buffer> data() const { return data_; }
-
- // Scalar append
- Status Append(bool val) {
- Reserve(1);
- BitUtil::SetBit(null_bitmap_data_, length_);
- if (val) {
- BitUtil::SetBit(raw_data_, length_);
- } else {
- BitUtil::ClearBit(raw_data_, length_);
- }
- ++length_;
- return Status::OK();
- }
-
- // Vector append
- //
- // If passed, valid_bytes is of equal length to values, and any zero byte
- // will be considered as a null for that slot
- Status Append(
- const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
-
- Status Finish(std::shared_ptr<Array>* out) override;
- Status Init(int32_t capacity) override;
-
- // Increase the capacity of the builder to accommodate at least the indicated
- // number of elements
- Status Resize(int32_t capacity) override;
-
- protected:
- std::shared_ptr<PoolBuffer> data_;
- uint8_t* raw_data_;
-};
-
-// gcc and clang disagree about how to handle template visibility when you have
-// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wattributes"
-#endif
-
-// Only instantiate these templates once
-extern template class ARROW_EXPORT NumericArray<Int8Type>;
-extern template class ARROW_EXPORT NumericArray<UInt8Type>;
-extern template class ARROW_EXPORT NumericArray<Int16Type>;
-extern template class ARROW_EXPORT NumericArray<UInt16Type>;
-extern template class ARROW_EXPORT NumericArray<Int32Type>;
-extern template class ARROW_EXPORT NumericArray<UInt32Type>;
-extern template class ARROW_EXPORT NumericArray<Int64Type>;
-extern template class ARROW_EXPORT NumericArray<UInt64Type>;
-extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
-extern template class ARROW_EXPORT NumericArray<FloatType>;
-extern template class ARROW_EXPORT NumericArray<DoubleType>;
-extern template class ARROW_EXPORT NumericArray<TimestampType>;
-
-#if defined(__GNUC__) && !defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_PRIMITIVE_H
[5/5] arrow git commit: ARROW-418: [C++] Array / Builder class code
reorganization, flattening
Posted by we...@apache.org.
ARROW-418: [C++] Array / Builder class code reorganization, flattening
I've been wanting to do this for a while -- it feels cleaner to me. I also am going to promote modules from arrow/util to the top level as well. I'm open to other ideas, too.
Author: Wes McKinney <we...@twosigma.com>
Closes #236 from wesm/ARROW-418 and squashes the following commits:
6f556ea [Wes McKinney] Add missing math.h include for clang
9dc2e22 [Wes McKinney] Fix remaining old includes
6f7ae77 [Wes McKinney] Fixes, cpplint
66ac3f7 [Wes McKinney] Promote buffer.h/status.h/memory-pool.h to top level directory
8cdf059 [Wes McKinney] Consolidate Array and Builder classes in array.h, builder.h. Remove arrow/types subdirectory
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/2c10d7cc
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/2c10d7cc
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/2c10d7cc
Branch: refs/heads/master
Commit: 2c10d7ccec3c07fb061e1988be16aecaf9916af4
Parents: 73fe556
Author: Wes McKinney <we...@twosigma.com>
Authored: Mon Dec 12 17:17:31 2016 -0500
Committer: Wes McKinney <we...@twosigma.com>
Committed: Mon Dec 12 17:17:31 2016 -0500
----------------------------------------------------------------------
cpp/CMakeLists.txt | 15 +-
cpp/src/arrow/CMakeLists.txt | 11 +
cpp/src/arrow/api.h | 13 +-
cpp/src/arrow/array-decimal-test.cc | 40 ++
cpp/src/arrow/array-list-test.cc | 237 ++++++++++++
cpp/src/arrow/array-primitive-test.cc | 476 +++++++++++++++++++++++
cpp/src/arrow/array-string-test.cc | 358 ++++++++++++++++++
cpp/src/arrow/array-struct-test.cc | 391 +++++++++++++++++++
cpp/src/arrow/array-test.cc | 5 +-
cpp/src/arrow/array.cc | 443 +++++++++++++++++++++-
cpp/src/arrow/array.h | 373 +++++++++++++++++-
cpp/src/arrow/buffer-test.cc | 140 +++++++
cpp/src/arrow/buffer.cc | 102 +++++
cpp/src/arrow/buffer.h | 232 ++++++++++++
cpp/src/arrow/builder.cc | 329 +++++++++++++++-
cpp/src/arrow/builder.h | 315 +++++++++++++++-
cpp/src/arrow/column-benchmark.cc | 4 +-
cpp/src/arrow/column-test.cc | 1 -
cpp/src/arrow/column.cc | 2 +-
cpp/src/arrow/io/file.cc | 6 +-
cpp/src/arrow/io/hdfs.cc | 6 +-
cpp/src/arrow/io/interfaces.cc | 4 +-
cpp/src/arrow/io/io-file-test.cc | 2 +-
cpp/src/arrow/io/io-hdfs-test.cc | 2 +-
cpp/src/arrow/io/libhdfs_shim.cc | 2 +-
cpp/src/arrow/io/memory.cc | 5 +-
cpp/src/arrow/io/test-common.h | 4 +-
cpp/src/arrow/ipc/adapter.cc | 9 +-
cpp/src/arrow/ipc/file.cc | 4 +-
cpp/src/arrow/ipc/ipc-adapter-test.cc | 10 +-
cpp/src/arrow/ipc/ipc-file-test.cc | 11 +-
cpp/src/arrow/ipc/ipc-json-test.cc | 18 +-
cpp/src/arrow/ipc/ipc-metadata-test.cc | 2 +-
cpp/src/arrow/ipc/json-integration-test.cc | 2 +-
cpp/src/arrow/ipc/json-internal.cc | 10 +-
cpp/src/arrow/ipc/json.cc | 6 +-
cpp/src/arrow/ipc/metadata-internal.cc | 4 +-
cpp/src/arrow/ipc/metadata.cc | 4 +-
cpp/src/arrow/ipc/test-common.h | 9 +-
cpp/src/arrow/ipc/util.h | 2 +-
cpp/src/arrow/memory_pool-test.cc | 69 ++++
cpp/src/arrow/memory_pool.cc | 111 ++++++
cpp/src/arrow/memory_pool.h | 43 +++
cpp/src/arrow/pretty_print-test.cc | 5 +-
cpp/src/arrow/pretty_print.cc | 5 +-
cpp/src/arrow/status-test.cc | 38 ++
cpp/src/arrow/status.cc | 86 +++++
cpp/src/arrow/status.h | 192 ++++++++++
cpp/src/arrow/table-test.cc | 4 +-
cpp/src/arrow/table.cc | 2 +-
cpp/src/arrow/test-util.h | 43 ++-
cpp/src/arrow/type.cc | 8 +-
cpp/src/arrow/type.h | 2 +-
cpp/src/arrow/types/CMakeLists.txt | 39 --
cpp/src/arrow/types/construct.cc | 124 ------
cpp/src/arrow/types/construct.h | 47 ---
cpp/src/arrow/types/datetime.h | 27 --
cpp/src/arrow/types/decimal-test.cc | 40 --
cpp/src/arrow/types/decimal.cc | 31 --
cpp/src/arrow/types/decimal.h | 28 --
cpp/src/arrow/types/list-test.cc | 241 ------------
cpp/src/arrow/types/list.cc | 162 --------
cpp/src/arrow/types/list.h | 170 ---------
cpp/src/arrow/types/primitive-test.cc | 478 ------------------------
cpp/src/arrow/types/primitive.cc | 294 ---------------
cpp/src/arrow/types/primitive.h | 371 ------------------
cpp/src/arrow/types/string-test.cc | 360 ------------------
cpp/src/arrow/types/string.cc | 150 --------
cpp/src/arrow/types/string.h | 149 --------
cpp/src/arrow/types/struct-test.cc | 396 --------------------
cpp/src/arrow/types/struct.cc | 108 ------
cpp/src/arrow/types/struct.h | 116 ------
cpp/src/arrow/types/test-common.h | 70 ----
cpp/src/arrow/types/union.cc | 27 --
cpp/src/arrow/types/union.h | 48 ---
cpp/src/arrow/util/CMakeLists.txt | 6 -
cpp/src/arrow/util/bit-util.cc | 4 +-
cpp/src/arrow/util/buffer-test.cc | 140 -------
cpp/src/arrow/util/buffer.cc | 102 -----
cpp/src/arrow/util/buffer.h | 232 ------------
cpp/src/arrow/util/memory-pool-test.cc | 69 ----
cpp/src/arrow/util/memory-pool.cc | 111 ------
cpp/src/arrow/util/memory-pool.h | 43 ---
cpp/src/arrow/util/status-test.cc | 38 --
cpp/src/arrow/util/status.cc | 86 -----
cpp/src/arrow/util/status.h | 192 ----------
python/src/pyarrow/adapters/builtin.cc | 2 +-
python/src/pyarrow/adapters/pandas.cc | 2 +-
python/src/pyarrow/common.cc | 4 +-
python/src/pyarrow/common.h | 5 +-
python/src/pyarrow/io.cc | 4 +-
91 files changed, 4103 insertions(+), 4630 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 798d75f..adcca0e 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -743,25 +743,17 @@ set(ARROW_PRIVATE_LINK_LIBS
set(ARROW_SRCS
src/arrow/array.cc
+ src/arrow/buffer.cc
src/arrow/builder.cc
src/arrow/column.cc
+ src/arrow/memory_pool.cc
src/arrow/pretty_print.cc
src/arrow/schema.cc
+ src/arrow/status.cc
src/arrow/table.cc
src/arrow/type.cc
- src/arrow/types/construct.cc
- src/arrow/types/decimal.cc
- src/arrow/types/list.cc
- src/arrow/types/primitive.cc
- src/arrow/types/string.cc
- src/arrow/types/struct.cc
- src/arrow/types/union.cc
-
src/arrow/util/bit-util.cc
- src/arrow/util/buffer.cc
- src/arrow/util/memory-pool.cc
- src/arrow/util/status.cc
)
add_library(arrow_objlib OBJECT
@@ -823,7 +815,6 @@ endif()
add_subdirectory(src/arrow)
add_subdirectory(src/arrow/io)
add_subdirectory(src/arrow/util)
-add_subdirectory(src/arrow/types)
#----------------------------------------------------------------------
# IPC library
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 6c0dea2..7d7bc29 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -20,9 +20,12 @@ install(FILES
api.h
array.h
column.h
+ buffer.h
builder.h
+ memory_pool.h
pretty_print.h
schema.h
+ status.h
table.h
type.h
type_fwd.h
@@ -37,9 +40,17 @@ install(FILES
set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
ADD_ARROW_TEST(array-test)
+ADD_ARROW_TEST(array-decimal-test)
+ADD_ARROW_TEST(array-list-test)
+ADD_ARROW_TEST(array-primitive-test)
+ADD_ARROW_TEST(array-string-test)
+ADD_ARROW_TEST(array-struct-test)
+ADD_ARROW_TEST(buffer-test)
ADD_ARROW_TEST(column-test)
+ADD_ARROW_TEST(memory_pool-test)
ADD_ARROW_TEST(pretty_print-test)
ADD_ARROW_TEST(schema-test)
+ADD_ARROW_TEST(status-test)
ADD_ARROW_TEST(table-test)
ADD_ARROW_BENCHMARK(column-benchmark)
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/api.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/api.h b/cpp/src/arrow/api.h
index 2d317b4..51437d8 100644
--- a/cpp/src/arrow/api.h
+++ b/cpp/src/arrow/api.h
@@ -21,20 +21,13 @@
#define ARROW_API_H
#include "arrow/array.h"
+#include "arrow/buffer.h"
#include "arrow/builder.h"
#include "arrow/column.h"
+#include "arrow/memory_pool.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
#endif // ARROW_API_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array-decimal-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
new file mode 100644
index 0000000..9e00fd9
--- /dev/null
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/type.h"
+
+namespace arrow {
+
+TEST(TypesTest, TestDecimalType) {
+ DecimalType t1(8, 4);
+
+ ASSERT_EQ(t1.type, Type::DECIMAL);
+ ASSERT_EQ(t1.precision, 8);
+ ASSERT_EQ(t1.scale, 4);
+
+ ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
+
+ // Test copy constructor
+ DecimalType t2 = t1;
+ ASSERT_EQ(t2.type, Type::DECIMAL);
+ ASSERT_EQ(t2.precision, 8);
+ ASSERT_EQ(t2.scale, 4);
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array-list-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-list-test.cc b/cpp/src/arrow/array-list-test.cc
new file mode 100644
index 0000000..8baaf06
--- /dev/null
+++ b/cpp/src/arrow/array-list-test.cc
@@ -0,0 +1,237 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+using std::shared_ptr;
+using std::string;
+using std::unique_ptr;
+using std::vector;
+
+namespace arrow {
+
+TEST(TypesTest, TestListType) {
+ std::shared_ptr<DataType> vt = std::make_shared<UInt8Type>();
+
+ ListType list_type(vt);
+ ASSERT_EQ(list_type.type, Type::LIST);
+
+ ASSERT_EQ(list_type.name(), string("list"));
+ ASSERT_EQ(list_type.ToString(), string("list<item: uint8>"));
+
+ ASSERT_EQ(list_type.value_type()->type, vt->type);
+ ASSERT_EQ(list_type.value_type()->type, vt->type);
+
+ std::shared_ptr<DataType> st = std::make_shared<StringType>();
+ std::shared_ptr<DataType> lt = std::make_shared<ListType>(st);
+ ASSERT_EQ(lt->ToString(), string("list<item: string>"));
+
+ ListType lt2(lt);
+ ASSERT_EQ(lt2.ToString(), string("list<item: list<item: string>>"));
+}
+
+// ----------------------------------------------------------------------
+// List tests
+
+class TestListBuilder : public TestBuilder {
+ public:
+ void SetUp() {
+ TestBuilder::SetUp();
+
+ value_type_ = TypePtr(new Int32Type());
+ type_ = TypePtr(new ListType(value_type_));
+
+ std::shared_ptr<ArrayBuilder> tmp;
+ ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+ builder_ = std::dynamic_pointer_cast<ListBuilder>(tmp);
+ }
+
+ void Done() {
+ std::shared_ptr<Array> out;
+ EXPECT_OK(builder_->Finish(&out));
+ result_ = std::dynamic_pointer_cast<ListArray>(out);
+ }
+
+ protected:
+ TypePtr value_type_;
+ TypePtr type_;
+
+ shared_ptr<ListBuilder> builder_;
+ shared_ptr<ListArray> result_;
+};
+
+TEST_F(TestListBuilder, Equality) {
+ Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+
+ ArrayPtr array, equal_array, unequal_array;
+ vector<int32_t> equal_offsets = {0, 1, 2, 5};
+ vector<int32_t> equal_values = {1, 2, 3, 4, 5, 2, 2, 2};
+ vector<int32_t> unequal_offsets = {0, 1, 4};
+ vector<int32_t> unequal_values = {1, 2, 2, 2, 3, 4, 5};
+
+ // setup two equal arrays
+ ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
+ ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
+
+ ASSERT_OK(builder_->Finish(&array));
+ ASSERT_OK(builder_->Append(equal_offsets.data(), equal_offsets.size()));
+ ASSERT_OK(vb->Append(equal_values.data(), equal_values.size()));
+
+ ASSERT_OK(builder_->Finish(&equal_array));
+ // now an unequal one
+ ASSERT_OK(builder_->Append(unequal_offsets.data(), unequal_offsets.size()));
+ ASSERT_OK(vb->Append(unequal_values.data(), unequal_values.size()));
+
+ ASSERT_OK(builder_->Finish(&unequal_array));
+
+ // Test array equality
+ EXPECT_TRUE(array->Equals(array));
+ EXPECT_TRUE(array->Equals(equal_array));
+ EXPECT_TRUE(equal_array->Equals(array));
+ EXPECT_FALSE(equal_array->Equals(unequal_array));
+ EXPECT_FALSE(unequal_array->Equals(equal_array));
+
+ // Test range equality
+ EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_array));
+ EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_array));
+ EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_array));
+ EXPECT_TRUE(array->RangeEquals(2, 3, 2, unequal_array));
+ EXPECT_TRUE(array->RangeEquals(3, 4, 1, unequal_array));
+}
+
+TEST_F(TestListBuilder, TestResize) {}
+
+TEST_F(TestListBuilder, TestAppendNull) {
+ ASSERT_OK(builder_->AppendNull());
+ ASSERT_OK(builder_->AppendNull());
+
+ Done();
+
+ ASSERT_OK(result_->Validate());
+ ASSERT_TRUE(result_->IsNull(0));
+ ASSERT_TRUE(result_->IsNull(1));
+
+ ASSERT_EQ(0, result_->raw_offsets()[0]);
+ ASSERT_EQ(0, result_->offset(1));
+ ASSERT_EQ(0, result_->offset(2));
+
+ Int32Array* values = static_cast<Int32Array*>(result_->values().get());
+ ASSERT_EQ(0, values->length());
+}
+
+void ValidateBasicListArray(const ListArray* result, const vector<int32_t>& values,
+ const vector<uint8_t>& is_valid) {
+ ASSERT_OK(result->Validate());
+ ASSERT_EQ(1, result->null_count());
+ ASSERT_EQ(0, result->values()->null_count());
+
+ ASSERT_EQ(3, result->length());
+ vector<int32_t> ex_offsets = {0, 3, 3, 7};
+ for (size_t i = 0; i < ex_offsets.size(); ++i) {
+ ASSERT_EQ(ex_offsets[i], result->offset(i));
+ }
+
+ for (int i = 0; i < result->length(); ++i) {
+ ASSERT_EQ(!static_cast<bool>(is_valid[i]), result->IsNull(i));
+ }
+
+ ASSERT_EQ(7, result->values()->length());
+ Int32Array* varr = static_cast<Int32Array*>(result->values().get());
+
+ for (size_t i = 0; i < values.size(); ++i) {
+ ASSERT_EQ(values[i], varr->Value(i));
+ }
+}
+
+TEST_F(TestListBuilder, TestBasics) {
+ vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+ vector<int> lengths = {3, 0, 4};
+ vector<uint8_t> is_valid = {1, 0, 1};
+
+ Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+
+ ASSERT_OK(builder_->Reserve(lengths.size()));
+ ASSERT_OK(vb->Reserve(values.size()));
+
+ int pos = 0;
+ for (size_t i = 0; i < lengths.size(); ++i) {
+ ASSERT_OK(builder_->Append(is_valid[i] > 0));
+ for (int j = 0; j < lengths[i]; ++j) {
+ vb->Append(values[pos++]);
+ }
+ }
+
+ Done();
+ ValidateBasicListArray(result_.get(), values, is_valid);
+}
+
+TEST_F(TestListBuilder, BulkAppend) {
+ vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+ vector<int> lengths = {3, 0, 4};
+ vector<uint8_t> is_valid = {1, 0, 1};
+ vector<int32_t> offsets = {0, 3, 3};
+
+ Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+ ASSERT_OK(vb->Reserve(values.size()));
+
+ builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+ for (int32_t value : values) {
+ vb->Append(value);
+ }
+ Done();
+ ValidateBasicListArray(result_.get(), values, is_valid);
+}
+
+TEST_F(TestListBuilder, BulkAppendInvalid) {
+ vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
+ vector<int> lengths = {3, 0, 4};
+ vector<uint8_t> is_null = {0, 1, 0};
+ vector<uint8_t> is_valid = {1, 0, 1};
+ vector<int32_t> offsets = {0, 2, 4}; // should be 0, 3, 3 given the is_null array
+
+ Int32Builder* vb = static_cast<Int32Builder*>(builder_->value_builder().get());
+ ASSERT_OK(vb->Reserve(values.size()));
+
+ builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+ builder_->Append(offsets.data(), offsets.size(), is_valid.data());
+ for (int32_t value : values) {
+ vb->Append(value);
+ }
+
+ Done();
+ ASSERT_RAISES(Invalid, result_->Validate());
+}
+
+TEST_F(TestListBuilder, TestZeroLength) {
+ // All buffers are null
+ Done();
+ ASSERT_OK(result_->Validate());
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array-primitive-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-primitive-test.cc b/cpp/src/arrow/array-primitive-test.cc
new file mode 100644
index 0000000..a10e240
--- /dev/null
+++ b/cpp/src/arrow/array-primitive-test.cc
@@ -0,0 +1,476 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
+#include "arrow/util/bit-util.h"
+
+using std::string;
+using std::shared_ptr;
+using std::unique_ptr;
+using std::vector;
+
+namespace arrow {
+
+class Array;
+
+#define PRIMITIVE_TEST(KLASS, ENUM, NAME) \
+ TEST(TypesTest, TestPrimitive_##ENUM) { \
+ KLASS tp; \
+ \
+ ASSERT_EQ(tp.type, Type::ENUM); \
+ ASSERT_EQ(tp.ToString(), string(NAME)); \
+ \
+ KLASS tp_copy = tp; \
+ ASSERT_EQ(tp_copy.type, Type::ENUM); \
+ }
+
+PRIMITIVE_TEST(Int8Type, INT8, "int8");
+PRIMITIVE_TEST(Int16Type, INT16, "int16");
+PRIMITIVE_TEST(Int32Type, INT32, "int32");
+PRIMITIVE_TEST(Int64Type, INT64, "int64");
+PRIMITIVE_TEST(UInt8Type, UINT8, "uint8");
+PRIMITIVE_TEST(UInt16Type, UINT16, "uint16");
+PRIMITIVE_TEST(UInt32Type, UINT32, "uint32");
+PRIMITIVE_TEST(UInt64Type, UINT64, "uint64");
+
+PRIMITIVE_TEST(FloatType, FLOAT, "float");
+PRIMITIVE_TEST(DoubleType, DOUBLE, "double");
+
+PRIMITIVE_TEST(BooleanType, BOOL, "bool");
+
+// ----------------------------------------------------------------------
+// Primitive type tests
+
+TEST_F(TestBuilder, TestReserve) {
+ builder_->Init(10);
+ ASSERT_EQ(2, builder_->null_bitmap()->size());
+
+ builder_->Reserve(30);
+ ASSERT_EQ(4, builder_->null_bitmap()->size());
+}
+
+template <typename Attrs>
+class TestPrimitiveBuilder : public TestBuilder {
+ public:
+ typedef typename Attrs::ArrayType ArrayType;
+ typedef typename Attrs::BuilderType BuilderType;
+ typedef typename Attrs::T T;
+ typedef typename Attrs::Type Type;
+
+ virtual void SetUp() {
+ TestBuilder::SetUp();
+
+ type_ = Attrs::type();
+
+ std::shared_ptr<ArrayBuilder> tmp;
+ ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+ builder_ = std::dynamic_pointer_cast<BuilderType>(tmp);
+
+ ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+ builder_nn_ = std::dynamic_pointer_cast<BuilderType>(tmp);
+ }
+
+ void RandomData(int N, double pct_null = 0.1) {
+ Attrs::draw(N, &draws_);
+
+ valid_bytes_.resize(N);
+ test::random_null_bytes(N, pct_null, valid_bytes_.data());
+ }
+
+ void Check(const std::shared_ptr<BuilderType>& builder, bool nullable) {
+ int size = builder->length();
+
+ auto ex_data = std::make_shared<Buffer>(
+ reinterpret_cast<uint8_t*>(draws_.data()), size * sizeof(T));
+
+ std::shared_ptr<Buffer> ex_null_bitmap;
+ int32_t ex_null_count = 0;
+
+ if (nullable) {
+ ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+ ex_null_count = test::null_count(valid_bytes_);
+ } else {
+ ex_null_bitmap = nullptr;
+ }
+
+ auto expected =
+ std::make_shared<ArrayType>(size, ex_data, ex_null_count, ex_null_bitmap);
+
+ std::shared_ptr<Array> out;
+ ASSERT_OK(builder->Finish(&out));
+
+ std::shared_ptr<ArrayType> result = std::dynamic_pointer_cast<ArrayType>(out);
+
+ // Builder is now reset
+ ASSERT_EQ(0, builder->length());
+ ASSERT_EQ(0, builder->capacity());
+ ASSERT_EQ(0, builder->null_count());
+ ASSERT_EQ(nullptr, builder->data());
+
+ ASSERT_EQ(ex_null_count, result->null_count());
+ ASSERT_TRUE(result->EqualsExact(*expected.get()));
+ }
+
+ protected:
+ std::shared_ptr<DataType> type_;
+ shared_ptr<BuilderType> builder_;
+ shared_ptr<BuilderType> builder_nn_;
+
+ vector<T> draws_;
+ vector<uint8_t> valid_bytes_;
+};
+
+#define PTYPE_DECL(CapType, c_type) \
+ typedef CapType##Array ArrayType; \
+ typedef CapType##Builder BuilderType; \
+ typedef CapType##Type Type; \
+ typedef c_type T; \
+ \
+ static std::shared_ptr<DataType> type() { \
+ return std::shared_ptr<DataType>(new Type()); \
+ }
+
+#define PINT_DECL(CapType, c_type, LOWER, UPPER) \
+ struct P##CapType { \
+ PTYPE_DECL(CapType, c_type); \
+ static void draw(int N, vector<T>* draws) { \
+ test::randint<T>(N, LOWER, UPPER, draws); \
+ } \
+ }
+
+#define PFLOAT_DECL(CapType, c_type, LOWER, UPPER) \
+ struct P##CapType { \
+ PTYPE_DECL(CapType, c_type); \
+ static void draw(int N, vector<T>* draws) { \
+ test::random_real<T>(N, 0, LOWER, UPPER, draws); \
+ } \
+ }
+
+PINT_DECL(UInt8, uint8_t, 0, UINT8_MAX);
+PINT_DECL(UInt16, uint16_t, 0, UINT16_MAX);
+PINT_DECL(UInt32, uint32_t, 0, UINT32_MAX);
+PINT_DECL(UInt64, uint64_t, 0, UINT64_MAX);
+
+PINT_DECL(Int8, int8_t, INT8_MIN, INT8_MAX);
+PINT_DECL(Int16, int16_t, INT16_MIN, INT16_MAX);
+PINT_DECL(Int32, int32_t, INT32_MIN, INT32_MAX);
+PINT_DECL(Int64, int64_t, INT64_MIN, INT64_MAX);
+
+PFLOAT_DECL(Float, float, -1000, 1000);
+PFLOAT_DECL(Double, double, -1000, 1000);
+
+struct PBoolean {
+ PTYPE_DECL(Boolean, uint8_t);
+};
+
+template <>
+void TestPrimitiveBuilder<PBoolean>::RandomData(int N, double pct_null) {
+ draws_.resize(N);
+ valid_bytes_.resize(N);
+
+ test::random_null_bytes(N, 0.5, draws_.data());
+ test::random_null_bytes(N, pct_null, valid_bytes_.data());
+}
+
+template <>
+void TestPrimitiveBuilder<PBoolean>::Check(
+ const std::shared_ptr<BooleanBuilder>& builder, bool nullable) {
+ int size = builder->length();
+
+ auto ex_data = test::bytes_to_null_buffer(draws_);
+
+ std::shared_ptr<Buffer> ex_null_bitmap;
+ int32_t ex_null_count = 0;
+
+ if (nullable) {
+ ex_null_bitmap = test::bytes_to_null_buffer(valid_bytes_);
+ ex_null_count = test::null_count(valid_bytes_);
+ } else {
+ ex_null_bitmap = nullptr;
+ }
+
+ auto expected =
+ std::make_shared<BooleanArray>(size, ex_data, ex_null_count, ex_null_bitmap);
+
+ std::shared_ptr<Array> out;
+ ASSERT_OK(builder->Finish(&out));
+ std::shared_ptr<BooleanArray> result = std::dynamic_pointer_cast<BooleanArray>(out);
+
+ // Builder is now reset
+ ASSERT_EQ(0, builder->length());
+ ASSERT_EQ(0, builder->capacity());
+ ASSERT_EQ(0, builder->null_count());
+ ASSERT_EQ(nullptr, builder->data());
+
+ ASSERT_EQ(ex_null_count, result->null_count());
+
+ ASSERT_EQ(expected->length(), result->length());
+
+ for (int i = 0; i < result->length(); ++i) {
+ if (nullable) { ASSERT_EQ(valid_bytes_[i] == 0, result->IsNull(i)) << i; }
+ bool actual = BitUtil::GetBit(result->raw_data(), i);
+ ASSERT_EQ(static_cast<bool>(draws_[i]), actual) << i;
+ }
+ ASSERT_TRUE(result->EqualsExact(*expected.get()));
+}
+
+typedef ::testing::Types<PBoolean, PUInt8, PUInt16, PUInt32, PUInt64, PInt8, PInt16,
+ PInt32, PInt64, PFloat, PDouble>
+ Primitives;
+
+TYPED_TEST_CASE(TestPrimitiveBuilder, Primitives);
+
+#define DECL_T() typedef typename TestFixture::T T;
+
+#define DECL_TYPE() typedef typename TestFixture::Type Type;
+
+#define DECL_ARRAYTYPE() typedef typename TestFixture::ArrayType ArrayType;
+
+TYPED_TEST(TestPrimitiveBuilder, TestInit) {
+ DECL_TYPE();
+
+ int n = 1000;
+ ASSERT_OK(this->builder_->Reserve(n));
+ ASSERT_EQ(BitUtil::NextPower2(n), this->builder_->capacity());
+ ASSERT_EQ(BitUtil::NextPower2(TypeTraits<Type>::bytes_required(n)),
+ this->builder_->data()->size());
+
+ // unsure if this should go in all builder classes
+ ASSERT_EQ(0, this->builder_->num_children());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendNull) {
+ int size = 1000;
+ for (int i = 0; i < size; ++i) {
+ ASSERT_OK(this->builder_->AppendNull());
+ }
+
+ std::shared_ptr<Array> result;
+ ASSERT_OK(this->builder_->Finish(&result));
+
+ for (int i = 0; i < size; ++i) {
+ ASSERT_TRUE(result->IsNull(i)) << i;
+ }
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestArrayDtorDealloc) {
+ DECL_T();
+
+ int size = 1000;
+
+ vector<T>& draws = this->draws_;
+ vector<uint8_t>& valid_bytes = this->valid_bytes_;
+
+ int64_t memory_before = this->pool_->bytes_allocated();
+
+ this->RandomData(size);
+
+ this->builder_->Reserve(size);
+
+ int i;
+ for (i = 0; i < size; ++i) {
+ if (valid_bytes[i] > 0) {
+ this->builder_->Append(draws[i]);
+ } else {
+ this->builder_->AppendNull();
+ }
+ }
+
+ do {
+ std::shared_ptr<Array> result;
+ ASSERT_OK(this->builder_->Finish(&result));
+ } while (false);
+
+ ASSERT_EQ(memory_before, this->pool_->bytes_allocated());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, Equality) {
+ DECL_T();
+
+ const int size = 1000;
+ this->RandomData(size);
+ vector<T>& draws = this->draws_;
+ vector<uint8_t>& valid_bytes = this->valid_bytes_;
+ ArrayPtr array, equal_array, unequal_array;
+ auto builder = this->builder_.get();
+ ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &array));
+ ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &equal_array));
+
+ // Make the not equal array by negating the first valid element with itself.
+ const auto first_valid = std::find_if(
+ valid_bytes.begin(), valid_bytes.end(), [](uint8_t valid) { return valid > 0; });
+ const int first_valid_idx = std::distance(valid_bytes.begin(), first_valid);
+ // This should be true with a very high probability, but might introduce flakiness
+ ASSERT_LT(first_valid_idx, size - 1);
+ draws[first_valid_idx] = ~*reinterpret_cast<int64_t*>(&draws[first_valid_idx]);
+ ASSERT_OK(MakeArray(valid_bytes, draws, size, builder, &unequal_array));
+
+ // test normal equality
+ EXPECT_TRUE(array->Equals(array));
+ EXPECT_TRUE(array->Equals(equal_array));
+ EXPECT_TRUE(equal_array->Equals(array));
+ EXPECT_FALSE(equal_array->Equals(unequal_array));
+ EXPECT_FALSE(unequal_array->Equals(equal_array));
+
+ // Test range equality
+ EXPECT_FALSE(array->RangeEquals(0, first_valid_idx + 1, 0, unequal_array));
+ EXPECT_FALSE(array->RangeEquals(first_valid_idx, size, first_valid_idx, unequal_array));
+ EXPECT_TRUE(array->RangeEquals(0, first_valid_idx, 0, unequal_array));
+ EXPECT_TRUE(
+ array->RangeEquals(first_valid_idx + 1, size, first_valid_idx + 1, unequal_array));
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendScalar) {
+ DECL_T();
+
+ const int size = 10000;
+
+ vector<T>& draws = this->draws_;
+ vector<uint8_t>& valid_bytes = this->valid_bytes_;
+
+ this->RandomData(size);
+
+ this->builder_->Reserve(1000);
+ this->builder_nn_->Reserve(1000);
+
+ int i;
+ int null_count = 0;
+ // Append the first 1000
+ for (i = 0; i < 1000; ++i) {
+ if (valid_bytes[i] > 0) {
+ this->builder_->Append(draws[i]);
+ } else {
+ this->builder_->AppendNull();
+ ++null_count;
+ }
+ this->builder_nn_->Append(draws[i]);
+ }
+
+ ASSERT_EQ(null_count, this->builder_->null_count());
+
+ ASSERT_EQ(1000, this->builder_->length());
+ ASSERT_EQ(1024, this->builder_->capacity());
+
+ ASSERT_EQ(1000, this->builder_nn_->length());
+ ASSERT_EQ(1024, this->builder_nn_->capacity());
+
+ this->builder_->Reserve(size - 1000);
+ this->builder_nn_->Reserve(size - 1000);
+
+ // Append the next 9000
+ for (i = 1000; i < size; ++i) {
+ if (valid_bytes[i] > 0) {
+ this->builder_->Append(draws[i]);
+ } else {
+ this->builder_->AppendNull();
+ }
+ this->builder_nn_->Append(draws[i]);
+ }
+
+ ASSERT_EQ(size, this->builder_->length());
+ ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
+
+ ASSERT_EQ(size, this->builder_nn_->length());
+ ASSERT_EQ(BitUtil::NextPower2(size), this->builder_nn_->capacity());
+
+ this->Check(this->builder_, true);
+ this->Check(this->builder_nn_, false);
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAppendVector) {
+ DECL_T();
+
+ int size = 10000;
+ this->RandomData(size);
+
+ vector<T>& draws = this->draws_;
+ vector<uint8_t>& valid_bytes = this->valid_bytes_;
+
+ // first slug
+ int K = 1000;
+
+ ASSERT_OK(this->builder_->Append(draws.data(), K, valid_bytes.data()));
+ ASSERT_OK(this->builder_nn_->Append(draws.data(), K));
+
+ ASSERT_EQ(1000, this->builder_->length());
+ ASSERT_EQ(1024, this->builder_->capacity());
+
+ ASSERT_EQ(1000, this->builder_nn_->length());
+ ASSERT_EQ(1024, this->builder_nn_->capacity());
+
+ // Append the next 9000
+ ASSERT_OK(this->builder_->Append(draws.data() + K, size - K, valid_bytes.data() + K));
+ ASSERT_OK(this->builder_nn_->Append(draws.data() + K, size - K));
+
+ ASSERT_EQ(size, this->builder_->length());
+ ASSERT_EQ(BitUtil::NextPower2(size), this->builder_->capacity());
+
+ this->Check(this->builder_, true);
+ this->Check(this->builder_nn_, false);
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestAdvance) {
+ int n = 1000;
+ ASSERT_OK(this->builder_->Reserve(n));
+
+ ASSERT_OK(this->builder_->Advance(100));
+ ASSERT_EQ(100, this->builder_->length());
+
+ ASSERT_OK(this->builder_->Advance(900));
+
+ int too_many = this->builder_->capacity() - 1000 + 1;
+ ASSERT_RAISES(Invalid, this->builder_->Advance(too_many));
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestResize) {
+ DECL_TYPE();
+
+ int cap = kMinBuilderCapacity * 2;
+
+ ASSERT_OK(this->builder_->Reserve(cap));
+ ASSERT_EQ(cap, this->builder_->capacity());
+
+ ASSERT_EQ(TypeTraits<Type>::bytes_required(cap), this->builder_->data()->size());
+ ASSERT_EQ(BitUtil::BytesForBits(cap), this->builder_->null_bitmap()->size());
+}
+
+TYPED_TEST(TestPrimitiveBuilder, TestReserve) {
+ ASSERT_OK(this->builder_->Reserve(10));
+ ASSERT_EQ(0, this->builder_->length());
+ ASSERT_EQ(kMinBuilderCapacity, this->builder_->capacity());
+
+ ASSERT_OK(this->builder_->Reserve(90));
+ ASSERT_OK(this->builder_->Advance(100));
+ ASSERT_OK(this->builder_->Reserve(kMinBuilderCapacity));
+
+ ASSERT_EQ(BitUtil::NextPower2(kMinBuilderCapacity + 100), this->builder_->capacity());
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array-string-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-string-test.cc b/cpp/src/arrow/array-string-test.cc
new file mode 100644
index 0000000..b144c63
--- /dev/null
+++ b/cpp/src/arrow/array-string-test.cc
@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+namespace arrow {
+
+class Buffer;
+
+TEST(TypesTest, BinaryType) {
+ BinaryType t1;
+ BinaryType e1;
+ StringType t2;
+ EXPECT_TRUE(t1.Equals(&e1));
+ EXPECT_FALSE(t1.Equals(&t2));
+ ASSERT_EQ(t1.type, Type::BINARY);
+ ASSERT_EQ(t1.ToString(), std::string("binary"));
+}
+
+TEST(TypesTest, TestStringType) {
+ StringType str;
+ ASSERT_EQ(str.type, Type::STRING);
+ ASSERT_EQ(str.ToString(), std::string("string"));
+}
+
+// ----------------------------------------------------------------------
+// String container
+
+class TestStringContainer : public ::testing::Test {
+ public:
+ void SetUp() {
+ chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
+ offsets_ = {0, 1, 1, 1, 3, 6};
+ valid_bytes_ = {1, 1, 0, 1, 1};
+ expected_ = {"a", "", "", "bb", "ccc"};
+
+ MakeArray();
+ }
+
+ void MakeArray() {
+ length_ = offsets_.size() - 1;
+ value_buf_ = test::GetBufferFromVector(chars_);
+ offsets_buf_ = test::GetBufferFromVector(offsets_);
+ null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+ null_count_ = test::null_count(valid_bytes_);
+
+ strings_ = std::make_shared<StringArray>(
+ length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+ }
+
+ protected:
+ std::vector<int32_t> offsets_;
+ std::vector<char> chars_;
+ std::vector<uint8_t> valid_bytes_;
+
+ std::vector<std::string> expected_;
+
+ std::shared_ptr<Buffer> value_buf_;
+ std::shared_ptr<Buffer> offsets_buf_;
+ std::shared_ptr<Buffer> null_bitmap_;
+
+ int null_count_;
+ int length_;
+
+ std::shared_ptr<StringArray> strings_;
+};
+
+TEST_F(TestStringContainer, TestArrayBasics) {
+ ASSERT_EQ(length_, strings_->length());
+ ASSERT_EQ(1, strings_->null_count());
+ ASSERT_OK(strings_->Validate());
+}
+
+TEST_F(TestStringContainer, TestType) {
+ TypePtr type = strings_->type();
+
+ ASSERT_EQ(Type::STRING, type->type);
+ ASSERT_EQ(Type::STRING, strings_->type_enum());
+}
+
+TEST_F(TestStringContainer, TestListFunctions) {
+ int pos = 0;
+ for (size_t i = 0; i < expected_.size(); ++i) {
+ ASSERT_EQ(pos, strings_->value_offset(i));
+ ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
+ pos += expected_[i].size();
+ }
+}
+
+TEST_F(TestStringContainer, TestDestructor) {
+ auto arr = std::make_shared<StringArray>(
+ length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+}
+
+TEST_F(TestStringContainer, TestGetString) {
+ for (size_t i = 0; i < expected_.size(); ++i) {
+ if (valid_bytes_[i] == 0) {
+ ASSERT_TRUE(strings_->IsNull(i));
+ } else {
+ ASSERT_EQ(expected_[i], strings_->GetString(i));
+ }
+ }
+}
+
+TEST_F(TestStringContainer, TestEmptyStringComparison) {
+ offsets_ = {0, 0, 0, 0, 0, 0};
+ offsets_buf_ = test::GetBufferFromVector(offsets_);
+ length_ = offsets_.size() - 1;
+
+ auto strings_a = std::make_shared<StringArray>(
+ length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
+ auto strings_b = std::make_shared<StringArray>(
+ length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
+ ASSERT_TRUE(strings_a->Equals(strings_b));
+}
+
+// ----------------------------------------------------------------------
+// String builder tests
+
+class TestStringBuilder : public TestBuilder {
+ public:
+ void SetUp() {
+ TestBuilder::SetUp();
+ type_ = TypePtr(new StringType());
+ builder_.reset(new StringBuilder(pool_, type_));
+ }
+
+ void Done() {
+ std::shared_ptr<Array> out;
+ EXPECT_OK(builder_->Finish(&out));
+
+ result_ = std::dynamic_pointer_cast<StringArray>(out);
+ result_->Validate();
+ }
+
+ protected:
+ TypePtr type_;
+
+ std::unique_ptr<StringBuilder> builder_;
+ std::shared_ptr<StringArray> result_;
+};
+
+TEST_F(TestStringBuilder, TestScalarAppend) {
+ std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
+ std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+
+ int N = strings.size();
+ int reps = 1000;
+
+ for (int j = 0; j < reps; ++j) {
+ for (int i = 0; i < N; ++i) {
+ if (is_null[i]) {
+ builder_->AppendNull();
+ } else {
+ builder_->Append(strings[i]);
+ }
+ }
+ }
+ Done();
+
+ ASSERT_EQ(reps * N, result_->length());
+ ASSERT_EQ(reps, result_->null_count());
+ ASSERT_EQ(reps * 6, result_->data()->size());
+
+ int32_t length;
+ int32_t pos = 0;
+ for (int i = 0; i < N * reps; ++i) {
+ if (is_null[i % N]) {
+ ASSERT_TRUE(result_->IsNull(i));
+ } else {
+ ASSERT_FALSE(result_->IsNull(i));
+ result_->GetValue(i, &length);
+ ASSERT_EQ(pos, result_->offset(i));
+ ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
+ ASSERT_EQ(strings[i % N], result_->GetString(i));
+
+ pos += length;
+ }
+ }
+}
+
+TEST_F(TestStringBuilder, TestZeroLength) {
+ // All buffers are null
+ Done();
+}
+
+// Binary container type
+// TODO(emkornfield) there should be some way to refactor these to avoid code duplicating
+// with String
+class TestBinaryContainer : public ::testing::Test {
+ public:
+ void SetUp() {
+ chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
+ offsets_ = {0, 1, 1, 1, 3, 6};
+ valid_bytes_ = {1, 1, 0, 1, 1};
+ expected_ = {"a", "", "", "bb", "ccc"};
+
+ MakeArray();
+ }
+
+ void MakeArray() {
+ length_ = offsets_.size() - 1;
+ value_buf_ = test::GetBufferFromVector(chars_);
+ offsets_buf_ = test::GetBufferFromVector(offsets_);
+
+ null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
+ null_count_ = test::null_count(valid_bytes_);
+
+ strings_ = std::make_shared<BinaryArray>(
+ length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+ }
+
+ protected:
+ std::vector<int32_t> offsets_;
+ std::vector<char> chars_;
+ std::vector<uint8_t> valid_bytes_;
+
+ std::vector<std::string> expected_;
+
+ std::shared_ptr<Buffer> value_buf_;
+ std::shared_ptr<Buffer> offsets_buf_;
+ std::shared_ptr<Buffer> null_bitmap_;
+
+ int null_count_;
+ int length_;
+
+ std::shared_ptr<BinaryArray> strings_;
+};
+
+TEST_F(TestBinaryContainer, TestArrayBasics) {
+ ASSERT_EQ(length_, strings_->length());
+ ASSERT_EQ(1, strings_->null_count());
+ ASSERT_OK(strings_->Validate());
+}
+
+TEST_F(TestBinaryContainer, TestType) {
+ TypePtr type = strings_->type();
+
+ ASSERT_EQ(Type::BINARY, type->type);
+ ASSERT_EQ(Type::BINARY, strings_->type_enum());
+}
+
+TEST_F(TestBinaryContainer, TestListFunctions) {
+ int pos = 0;
+ for (size_t i = 0; i < expected_.size(); ++i) {
+ ASSERT_EQ(pos, strings_->value_offset(i));
+ ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
+ pos += expected_[i].size();
+ }
+}
+
+TEST_F(TestBinaryContainer, TestDestructor) {
+ auto arr = std::make_shared<BinaryArray>(
+ length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
+}
+
+TEST_F(TestBinaryContainer, TestGetValue) {
+ for (size_t i = 0; i < expected_.size(); ++i) {
+ if (valid_bytes_[i] == 0) {
+ ASSERT_TRUE(strings_->IsNull(i));
+ } else {
+ int32_t len = -1;
+ const uint8_t* bytes = strings_->GetValue(i, &len);
+ ASSERT_EQ(0, std::memcmp(expected_[i].data(), bytes, len));
+ }
+ }
+}
+
+class TestBinaryBuilder : public TestBuilder {
+ public:
+ void SetUp() {
+ TestBuilder::SetUp();
+ type_ = TypePtr(new BinaryType());
+ builder_.reset(new BinaryBuilder(pool_, type_));
+ }
+
+ void Done() {
+ std::shared_ptr<Array> out;
+ EXPECT_OK(builder_->Finish(&out));
+
+ result_ = std::dynamic_pointer_cast<BinaryArray>(out);
+ result_->Validate();
+ }
+
+ protected:
+ TypePtr type_;
+
+ std::unique_ptr<BinaryBuilder> builder_;
+ std::shared_ptr<BinaryArray> result_;
+};
+
+TEST_F(TestBinaryBuilder, TestScalarAppend) {
+ std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
+ std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
+
+ int N = strings.size();
+ int reps = 1000;
+
+ for (int j = 0; j < reps; ++j) {
+ for (int i = 0; i < N; ++i) {
+ if (is_null[i]) {
+ builder_->AppendNull();
+ } else {
+ builder_->Append(
+ reinterpret_cast<const uint8_t*>(strings[i].data()), strings[i].size());
+ }
+ }
+ }
+ Done();
+ ASSERT_OK(result_->Validate());
+ ASSERT_EQ(reps * N, result_->length());
+ ASSERT_EQ(reps, result_->null_count());
+ ASSERT_EQ(reps * 6, result_->data()->size());
+
+ int32_t length;
+ for (int i = 0; i < N * reps; ++i) {
+ if (is_null[i % N]) {
+ ASSERT_TRUE(result_->IsNull(i));
+ } else {
+ ASSERT_FALSE(result_->IsNull(i));
+ const uint8_t* vals = result_->GetValue(i, &length);
+ ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
+ ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length));
+ }
+ }
+}
+
+TEST_F(TestBinaryBuilder, TestZeroLength) {
+ // All buffers are null
+ Done();
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array-struct-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-struct-test.cc b/cpp/src/arrow/array-struct-test.cc
new file mode 100644
index 0000000..58386fe
--- /dev/null
+++ b/cpp/src/arrow/array-struct-test.cc
@@ -0,0 +1,391 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+#include "arrow/type.h"
+
+using std::shared_ptr;
+using std::string;
+using std::vector;
+
+namespace arrow {
+
+TEST(TestStructType, Basics) {
+ TypePtr f0_type = TypePtr(new Int32Type());
+ auto f0 = std::make_shared<Field>("f0", f0_type);
+
+ TypePtr f1_type = TypePtr(new StringType());
+ auto f1 = std::make_shared<Field>("f1", f1_type);
+
+ TypePtr f2_type = TypePtr(new UInt8Type());
+ auto f2 = std::make_shared<Field>("f2", f2_type);
+
+ vector<shared_ptr<Field>> fields = {f0, f1, f2};
+
+ StructType struct_type(fields);
+
+ ASSERT_TRUE(struct_type.child(0)->Equals(f0));
+ ASSERT_TRUE(struct_type.child(1)->Equals(f1));
+ ASSERT_TRUE(struct_type.child(2)->Equals(f2));
+
+ ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
+
+ // TODO(wesm): out of bounds for field(...)
+}
+
+void ValidateBasicStructArray(const StructArray* result,
+ const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
+ const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
+ const vector<int>& list_offsets, const vector<int32_t>& int_values) {
+ ASSERT_EQ(4, result->length());
+ ASSERT_OK(result->Validate());
+
+ auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
+ auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
+ auto int32_arr = static_cast<Int32Array*>(result->field(1).get());
+
+ ASSERT_EQ(0, result->null_count());
+ ASSERT_EQ(1, list_char_arr->null_count());
+ ASSERT_EQ(0, int32_arr->null_count());
+
+ // List<char>
+ ASSERT_EQ(4, list_char_arr->length());
+ ASSERT_EQ(10, list_char_arr->values()->length());
+ for (size_t i = 0; i < list_offsets.size(); ++i) {
+ ASSERT_EQ(list_offsets[i], list_char_arr->raw_offsets()[i]);
+ }
+ for (size_t i = 0; i < list_values.size(); ++i) {
+ ASSERT_EQ(list_values[i], char_arr->Value(i));
+ }
+
+ // Int32
+ ASSERT_EQ(4, int32_arr->length());
+ for (size_t i = 0; i < int_values.size(); ++i) {
+ ASSERT_EQ(int_values[i], int32_arr->Value(i));
+ }
+}
+
+// ----------------------------------------------------------------------------------
+// Struct test
+class TestStructBuilder : public TestBuilder {
+ public:
+ void SetUp() {
+ TestBuilder::SetUp();
+
+ auto int32_type = TypePtr(new Int32Type());
+ auto char_type = TypePtr(new Int8Type());
+ auto list_type = TypePtr(new ListType(char_type));
+
+ std::vector<TypePtr> types = {list_type, int32_type};
+ std::vector<FieldPtr> fields;
+ fields.push_back(FieldPtr(new Field("list", list_type)));
+ fields.push_back(FieldPtr(new Field("int", int32_type)));
+
+ type_ = TypePtr(new StructType(fields));
+ value_fields_ = fields;
+
+ std::shared_ptr<ArrayBuilder> tmp;
+ ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
+
+ builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
+ ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+ }
+
+ void Done() {
+ std::shared_ptr<Array> out;
+ ASSERT_OK(builder_->Finish(&out));
+ result_ = std::dynamic_pointer_cast<StructArray>(out);
+ }
+
+ protected:
+ std::vector<FieldPtr> value_fields_;
+ TypePtr type_;
+
+ std::shared_ptr<StructBuilder> builder_;
+ std::shared_ptr<StructArray> result_;
+};
+
+TEST_F(TestStructBuilder, TestAppendNull) {
+ ASSERT_OK(builder_->AppendNull());
+ ASSERT_OK(builder_->AppendNull());
+ ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+
+ ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+ ASSERT_OK(list_vb->AppendNull());
+ ASSERT_OK(list_vb->AppendNull());
+ ASSERT_EQ(2, list_vb->length());
+
+ Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+ ASSERT_OK(int_vb->AppendNull());
+ ASSERT_OK(int_vb->AppendNull());
+ ASSERT_EQ(2, int_vb->length());
+
+ Done();
+
+ ASSERT_OK(result_->Validate());
+
+ ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
+ ASSERT_EQ(2, result_->length());
+ ASSERT_EQ(2, result_->field(0)->length());
+ ASSERT_EQ(2, result_->field(1)->length());
+ ASSERT_TRUE(result_->IsNull(0));
+ ASSERT_TRUE(result_->IsNull(1));
+ ASSERT_TRUE(result_->field(0)->IsNull(0));
+ ASSERT_TRUE(result_->field(0)->IsNull(1));
+ ASSERT_TRUE(result_->field(1)->IsNull(0));
+ ASSERT_TRUE(result_->field(1)->IsNull(1));
+
+ ASSERT_EQ(Type::LIST, result_->field(0)->type_enum());
+ ASSERT_EQ(Type::INT32, result_->field(1)->type_enum());
+}
+
+TEST_F(TestStructBuilder, TestBasics) {
+ vector<int32_t> int_values = {1, 2, 3, 4};
+ vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+ vector<int> list_lengths = {3, 0, 3, 4};
+ vector<int> list_offsets = {0, 3, 3, 6, 10};
+ vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+ vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+ ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+ Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+ Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+ ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
+
+ EXPECT_OK(builder_->Resize(list_lengths.size()));
+ EXPECT_OK(char_vb->Resize(list_values.size()));
+ EXPECT_OK(int_vb->Resize(int_values.size()));
+
+ int pos = 0;
+ for (size_t i = 0; i < list_lengths.size(); ++i) {
+ ASSERT_OK(list_vb->Append(list_is_valid[i] > 0));
+ int_vb->UnsafeAppend(int_values[i]);
+ for (int j = 0; j < list_lengths[i]; ++j) {
+ char_vb->UnsafeAppend(list_values[pos++]);
+ }
+ }
+
+ for (size_t i = 0; i < struct_is_valid.size(); ++i) {
+ ASSERT_OK(builder_->Append(struct_is_valid[i] > 0));
+ }
+
+ Done();
+
+ ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
+ list_lengths, list_offsets, int_values);
+}
+
+TEST_F(TestStructBuilder, BulkAppend) {
+ vector<int32_t> int_values = {1, 2, 3, 4};
+ vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+ vector<int> list_lengths = {3, 0, 3, 4};
+ vector<int> list_offsets = {0, 3, 3, 6};
+ vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+ vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+ ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+ Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+ Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+
+ ASSERT_OK(builder_->Resize(list_lengths.size()));
+ ASSERT_OK(char_vb->Resize(list_values.size()));
+ ASSERT_OK(int_vb->Resize(int_values.size()));
+
+ builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+
+ list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+ for (int8_t value : list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ Done();
+ ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
+ list_lengths, list_offsets, int_values);
+}
+
+TEST_F(TestStructBuilder, BulkAppendInvalid) {
+ vector<int32_t> int_values = {1, 2, 3, 4};
+ vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+ vector<int> list_lengths = {3, 0, 3, 4};
+ vector<int> list_offsets = {0, 3, 3, 6};
+ vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+ vector<uint8_t> struct_is_valid = {1, 0, 1, 1}; // should be 1, 1, 1, 1
+
+ ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+ Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+ Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+
+ ASSERT_OK(builder_->Reserve(list_lengths.size()));
+ ASSERT_OK(char_vb->Reserve(list_values.size()));
+ ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+ builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+
+ list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+ for (int8_t value : list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ Done();
+ // Even null bitmap of the parent Struct is not valid, Validate() will ignore it.
+ ASSERT_OK(result_->Validate());
+}
+
+TEST_F(TestStructBuilder, TestEquality) {
+ ArrayPtr array, equal_array;
+ ArrayPtr unequal_bitmap_array, unequal_offsets_array, unequal_values_array;
+
+ vector<int32_t> int_values = {1, 2, 3, 4};
+ vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
+ vector<int> list_lengths = {3, 0, 3, 4};
+ vector<int> list_offsets = {0, 3, 3, 6};
+ vector<uint8_t> list_is_valid = {1, 0, 1, 1};
+ vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
+
+ vector<int32_t> unequal_int_values = {4, 2, 3, 1};
+ vector<char> unequal_list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'l', 'u', 'c', 'y'};
+ vector<int> unequal_list_offsets = {0, 3, 4, 6};
+ vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
+ vector<uint8_t> unequal_struct_is_valid = {1, 0, 0, 1};
+
+ ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
+ Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
+ Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
+ ASSERT_OK(builder_->Reserve(list_lengths.size()));
+ ASSERT_OK(char_vb->Reserve(list_values.size()));
+ ASSERT_OK(int_vb->Reserve(int_values.size()));
+
+ // setup two equal arrays, one of which takes an unequal bitmap
+ builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+ list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+ for (int8_t value : list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ ASSERT_OK(builder_->Finish(&array));
+
+ ASSERT_OK(builder_->Resize(list_lengths.size()));
+ ASSERT_OK(char_vb->Resize(list_values.size()));
+ ASSERT_OK(int_vb->Resize(int_values.size()));
+
+ builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+ list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+ for (int8_t value : list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ ASSERT_OK(builder_->Finish(&equal_array));
+
+ ASSERT_OK(builder_->Resize(list_lengths.size()));
+ ASSERT_OK(char_vb->Resize(list_values.size()));
+ ASSERT_OK(int_vb->Resize(int_values.size()));
+
+ // setup an unequal one with the unequal bitmap
+ builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data());
+ list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+ for (int8_t value : list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ ASSERT_OK(builder_->Finish(&unequal_bitmap_array));
+
+ ASSERT_OK(builder_->Resize(list_lengths.size()));
+ ASSERT_OK(char_vb->Resize(list_values.size()));
+ ASSERT_OK(int_vb->Resize(int_values.size()));
+
+ // setup an unequal one with unequal offsets
+ builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+ list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
+ unequal_list_is_valid.data());
+ for (int8_t value : list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ ASSERT_OK(builder_->Finish(&unequal_offsets_array));
+
+ ASSERT_OK(builder_->Resize(list_lengths.size()));
+ ASSERT_OK(char_vb->Resize(list_values.size()));
+ ASSERT_OK(int_vb->Resize(int_values.size()));
+
+ // setup anunequal one with unequal values
+ builder_->Append(struct_is_valid.size(), struct_is_valid.data());
+ list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
+ for (int8_t value : unequal_list_values) {
+ char_vb->UnsafeAppend(value);
+ }
+ for (int32_t value : unequal_int_values) {
+ int_vb->UnsafeAppend(value);
+ }
+
+ ASSERT_OK(builder_->Finish(&unequal_values_array));
+
+ // Test array equality
+ EXPECT_TRUE(array->Equals(array));
+ EXPECT_TRUE(array->Equals(equal_array));
+ EXPECT_TRUE(equal_array->Equals(array));
+ EXPECT_FALSE(equal_array->Equals(unequal_bitmap_array));
+ EXPECT_FALSE(unequal_bitmap_array->Equals(equal_array));
+ EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_values_array));
+ EXPECT_FALSE(unequal_values_array->Equals(unequal_bitmap_array));
+ EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_offsets_array));
+ EXPECT_FALSE(unequal_offsets_array->Equals(unequal_bitmap_array));
+
+ // Test range equality
+ EXPECT_TRUE(array->RangeEquals(0, 4, 0, equal_array));
+ EXPECT_TRUE(array->RangeEquals(3, 4, 3, unequal_bitmap_array));
+ EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_offsets_array));
+ EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_offsets_array));
+ EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_offsets_array));
+ EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
+ EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
+ EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
+}
+
+TEST_F(TestStructBuilder, TestZeroLength) {
+ // All buffers are null
+ Done();
+ ASSERT_OK(result_->Validate());
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 1581244..783104e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -24,11 +24,10 @@
#include "gtest/gtest.h"
#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index 1f0bb66..7ab61f5 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -19,10 +19,13 @@
#include <cstdint>
#include <cstring>
+#include <sstream>
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/util/logging.h"
namespace arrow {
@@ -85,4 +88,440 @@ Status NullArray::Accept(ArrayVisitor* visitor) const {
return visitor->Visit(*this);
}
+// ----------------------------------------------------------------------
+// Primitive array base
+
+PrimitiveArray::PrimitiveArray(const TypePtr& type, int32_t length,
+ const std::shared_ptr<Buffer>& data, int32_t null_count,
+ const std::shared_ptr<Buffer>& null_bitmap)
+ : Array(type, length, null_count, null_bitmap) {
+ data_ = data;
+ raw_data_ = data == nullptr ? nullptr : data_->data();
+}
+
+bool PrimitiveArray::EqualsExact(const PrimitiveArray& other) const {
+ if (this == &other) { return true; }
+ if (null_count_ != other.null_count_) { return false; }
+
+ if (null_count_ > 0) {
+ bool equal_bitmap =
+ null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+ if (!equal_bitmap) { return false; }
+
+ const uint8_t* this_data = raw_data_;
+ const uint8_t* other_data = other.raw_data_;
+
+ auto size_meta = dynamic_cast<const FixedWidthType*>(type_.get());
+ int value_byte_size = size_meta->bit_width() / 8;
+ DCHECK_GT(value_byte_size, 0);
+
+ for (int i = 0; i < length_; ++i) {
+ if (!IsNull(i) && memcmp(this_data, other_data, value_byte_size)) { return false; }
+ this_data += value_byte_size;
+ other_data += value_byte_size;
+ }
+ return true;
+ } else {
+ if (length_ == 0 && other.length_ == 0) { return true; }
+ return data_->Equals(*other.data_, length_);
+ }
+}
+
+bool PrimitiveArray::Equals(const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ return EqualsExact(*static_cast<const PrimitiveArray*>(arr.get()));
+}
+
+template <typename T>
+Status NumericArray<T>::Accept(ArrayVisitor* visitor) const {
+ return visitor->Visit(*this);
+}
+
+template class NumericArray<UInt8Type>;
+template class NumericArray<UInt16Type>;
+template class NumericArray<UInt32Type>;
+template class NumericArray<UInt64Type>;
+template class NumericArray<Int8Type>;
+template class NumericArray<Int16Type>;
+template class NumericArray<Int32Type>;
+template class NumericArray<Int64Type>;
+template class NumericArray<TimestampType>;
+template class NumericArray<HalfFloatType>;
+template class NumericArray<FloatType>;
+template class NumericArray<DoubleType>;
+
+// ----------------------------------------------------------------------
+// BooleanArray
+
+BooleanArray::BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
+ int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+ : PrimitiveArray(
+ std::make_shared<BooleanType>(), length, data, null_count, null_bitmap) {}
+
+BooleanArray::BooleanArray(const TypePtr& type, int32_t length,
+ const std::shared_ptr<Buffer>& data, int32_t null_count,
+ const std::shared_ptr<Buffer>& null_bitmap)
+ : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
+bool BooleanArray::EqualsExact(const BooleanArray& other) const {
+ if (this == &other) return true;
+ if (null_count_ != other.null_count_) { return false; }
+
+ if (null_count_ > 0) {
+ bool equal_bitmap =
+ null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
+ if (!equal_bitmap) { return false; }
+
+ const uint8_t* this_data = raw_data_;
+ const uint8_t* other_data = other.raw_data_;
+
+ for (int i = 0; i < length_; ++i) {
+ if (!IsNull(i) && BitUtil::GetBit(this_data, i) != BitUtil::GetBit(other_data, i)) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ return data_->Equals(*other.data_, BitUtil::BytesForBits(length_));
+ }
+}
+
+bool BooleanArray::Equals(const ArrayPtr& arr) const {
+ if (this == arr.get()) return true;
+ if (Type::BOOL != arr->type_enum()) { return false; }
+ return EqualsExact(*static_cast<const BooleanArray*>(arr.get()));
+}
+
+bool BooleanArray::RangeEquals(int32_t start_idx, int32_t end_idx,
+ int32_t other_start_idx, const ArrayPtr& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ const auto other = static_cast<BooleanArray*>(arr.get());
+ for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+ const bool is_null = IsNull(i);
+ if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
+ return false;
+ }
+ }
+ return true;
+}
+
+Status BooleanArray::Accept(ArrayVisitor* visitor) const {
+ return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+// ListArray
+
+bool ListArray::EqualsExact(const ListArray& other) const {
+ if (this == &other) { return true; }
+ if (null_count_ != other.null_count_) { return false; }
+
+ bool equal_offsets =
+ offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+ if (!equal_offsets) { return false; }
+ bool equal_null_bitmap = true;
+ if (null_count_ > 0) {
+ equal_null_bitmap =
+ null_bitmap_->Equals(*other.null_bitmap_, BitUtil::BytesForBits(length_));
+ }
+
+ if (!equal_null_bitmap) { return false; }
+
+ return values()->Equals(other.values());
+}
+
+bool ListArray::Equals(const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ return EqualsExact(*static_cast<const ListArray*>(arr.get()));
+}
+
+bool ListArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ const auto other = static_cast<ListArray*>(arr.get());
+ for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+ const bool is_null = IsNull(i);
+ if (is_null != arr->IsNull(o_i)) { return false; }
+ if (is_null) continue;
+ const int32_t begin_offset = offset(i);
+ const int32_t end_offset = offset(i + 1);
+ const int32_t other_begin_offset = other->offset(o_i);
+ const int32_t other_end_offset = other->offset(o_i + 1);
+ // Underlying can't be equal if the size isn't equal
+ if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
+ return false;
+ }
+ if (!values_->RangeEquals(
+ begin_offset, end_offset, other_begin_offset, other->values())) {
+ return false;
+ }
+ }
+ return true;
+}
+
+Status ListArray::Validate() const {
+ if (length_ < 0) { return Status::Invalid("Length was negative"); }
+ if (!offset_buffer_) { return Status::Invalid("offset_buffer_ was null"); }
+ if (offset_buffer_->size() / static_cast<int>(sizeof(int32_t)) < length_) {
+ std::stringstream ss;
+ ss << "offset buffer size (bytes): " << offset_buffer_->size()
+ << " isn't large enough for length: " << length_;
+ return Status::Invalid(ss.str());
+ }
+ const int32_t last_offset = offset(length_);
+ if (last_offset > 0) {
+ if (!values_) {
+ return Status::Invalid("last offset was non-zero and values was null");
+ }
+ if (values_->length() != last_offset) {
+ std::stringstream ss;
+ ss << "Final offset invariant not equal to values length: " << last_offset
+ << "!=" << values_->length();
+ return Status::Invalid(ss.str());
+ }
+
+ const Status child_valid = values_->Validate();
+ if (!child_valid.ok()) {
+ std::stringstream ss;
+ ss << "Child array invalid: " << child_valid.ToString();
+ return Status::Invalid(ss.str());
+ }
+ }
+
+ int32_t prev_offset = offset(0);
+ if (prev_offset != 0) { return Status::Invalid("The first offset wasn't zero"); }
+ for (int32_t i = 1; i <= length_; ++i) {
+ int32_t current_offset = offset(i);
+ if (IsNull(i - 1) && current_offset != prev_offset) {
+ std::stringstream ss;
+ ss << "Offset invariant failure at: " << i << " inconsistent offsets for null slot"
+ << current_offset << "!=" << prev_offset;
+ return Status::Invalid(ss.str());
+ }
+ if (current_offset < prev_offset) {
+ std::stringstream ss;
+ ss << "Offset invariant failure: " << i
+ << " inconsistent offset for non-null slot: " << current_offset << "<"
+ << prev_offset;
+ return Status::Invalid(ss.str());
+ }
+ prev_offset = current_offset;
+ }
+ return Status::OK();
+}
+
+Status ListArray::Accept(ArrayVisitor* visitor) const {
+ return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+// String and binary
+
+static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
+static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
+
+BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+ const std::shared_ptr<Buffer>& data, int32_t null_count,
+ const std::shared_ptr<Buffer>& null_bitmap)
+ : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
+
+BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
+ const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
+ int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
+ : Array(type, length, null_count, null_bitmap),
+ offset_buffer_(offsets),
+ offsets_(reinterpret_cast<const int32_t*>(offset_buffer_->data())),
+ data_buffer_(data),
+ data_(nullptr) {
+ if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
+}
+
+Status BinaryArray::Validate() const {
+ // TODO(wesm): what to do here?
+ return Status::OK();
+}
+
+bool BinaryArray::EqualsExact(const BinaryArray& other) const {
+ if (!Array::EqualsExact(other)) { return false; }
+
+ bool equal_offsets =
+ offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
+ if (!equal_offsets) { return false; }
+
+ if (!data_buffer_ && !(other.data_buffer_)) { return true; }
+
+ return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
+}
+
+bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ return EqualsExact(*static_cast<const BinaryArray*>(arr.get()));
+}
+
+bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ const auto other = static_cast<const BinaryArray*>(arr.get());
+ for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+ const bool is_null = IsNull(i);
+ if (is_null != arr->IsNull(o_i)) { return false; }
+ if (is_null) continue;
+ const int32_t begin_offset = offset(i);
+ const int32_t end_offset = offset(i + 1);
+ const int32_t other_begin_offset = other->offset(o_i);
+ const int32_t other_end_offset = other->offset(o_i + 1);
+ // Underlying can't be equal if the size isn't equal
+ if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
+ return false;
+ }
+
+ if (std::memcmp(data_ + begin_offset, other->data_ + other_begin_offset,
+ end_offset - begin_offset)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+Status BinaryArray::Accept(ArrayVisitor* visitor) const {
+ return visitor->Visit(*this);
+}
+
+StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+ const std::shared_ptr<Buffer>& data, int32_t null_count,
+ const std::shared_ptr<Buffer>& null_bitmap)
+ : BinaryArray(kString, length, offsets, data, null_count, null_bitmap) {}
+
+Status StringArray::Validate() const {
+ // TODO(emkornfield) Validate proper UTF8 code points?
+ return BinaryArray::Validate();
+}
+
+Status StringArray::Accept(ArrayVisitor* visitor) const {
+ return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+// Struct
+
+std::shared_ptr<Array> StructArray::field(int32_t pos) const {
+ DCHECK_GT(field_arrays_.size(), 0);
+ return field_arrays_[pos];
+}
+
+bool StructArray::Equals(const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ if (null_count_ != arr->null_count()) { return false; }
+ return RangeEquals(0, length_, 0, arr);
+}
+
+bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (Type::STRUCT != arr->type_enum()) { return false; }
+ const auto other = static_cast<StructArray*>(arr.get());
+
+ bool equal_fields = true;
+ for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+ if (IsNull(i) != arr->IsNull(o_i)) { return false; }
+ if (IsNull(i)) continue;
+ for (size_t j = 0; j < field_arrays_.size(); ++j) {
+ // TODO: really we should be comparing stretches of non-null data rather
+ // than looking at one value at a time.
+ equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other->field(j));
+ if (!equal_fields) { return false; }
+ }
+ }
+
+ return true;
+}
+
+Status StructArray::Validate() const {
+ if (length_ < 0) { return Status::Invalid("Length was negative"); }
+
+ if (null_count() > length_) {
+ return Status::Invalid("Null count exceeds the length of this struct");
+ }
+
+ if (field_arrays_.size() > 0) {
+ // Validate fields
+ int32_t array_length = field_arrays_[0]->length();
+ size_t idx = 0;
+ for (auto it : field_arrays_) {
+ if (it->length() != array_length) {
+ std::stringstream ss;
+ ss << "Length is not equal from field " << it->type()->ToString()
+ << " at position {" << idx << "}";
+ return Status::Invalid(ss.str());
+ }
+
+ const Status child_valid = it->Validate();
+ if (!child_valid.ok()) {
+ std::stringstream ss;
+ ss << "Child array invalid: " << child_valid.ToString() << " at position {" << idx
+ << "}";
+ return Status::Invalid(ss.str());
+ }
+ ++idx;
+ }
+
+ if (array_length > 0 && array_length != length_) {
+ return Status::Invalid("Struct's length is not equal to its child arrays");
+ }
+ }
+ return Status::OK();
+}
+
+Status StructArray::Accept(ArrayVisitor* visitor) const {
+ return visitor->Visit(*this);
+}
+
+// ----------------------------------------------------------------------
+
+#define MAKE_PRIMITIVE_ARRAY_CASE(ENUM, ArrayType) \
+ case Type::ENUM: \
+ out->reset(new ArrayType(type, length, data, null_count, null_bitmap)); \
+ break;
+
+Status MakePrimitiveArray(const TypePtr& type, int32_t length,
+ const std::shared_ptr<Buffer>& data, int32_t null_count,
+ const std::shared_ptr<Buffer>& null_bitmap, ArrayPtr* out) {
+ switch (type->type) {
+ MAKE_PRIMITIVE_ARRAY_CASE(BOOL, BooleanArray);
+ MAKE_PRIMITIVE_ARRAY_CASE(UINT8, UInt8Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(INT8, Int8Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(UINT16, UInt16Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(INT16, Int16Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(UINT32, UInt32Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(INT32, Int32Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(UINT64, UInt64Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(INT64, Int64Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(FLOAT, FloatArray);
+ MAKE_PRIMITIVE_ARRAY_CASE(DOUBLE, DoubleArray);
+ MAKE_PRIMITIVE_ARRAY_CASE(TIME, Int64Array);
+ MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP, TimestampArray);
+ MAKE_PRIMITIVE_ARRAY_CASE(TIMESTAMP_DOUBLE, DoubleArray);
+ default:
+ return Status::NotImplemented(type->ToString());
+ }
+#ifdef NDEBUG
+ return Status::OK();
+#else
+ return (*out)->Validate();
+#endif
+}
+
} // namespace arrow
[4/5] arrow git commit: ARROW-418: [C++] Array / Builder class code
reorganization, flattening
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/array.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 91fb93e..1a4a923 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -18,9 +18,13 @@
#ifndef ARROW_ARRAY_H
#define ARROW_ARRAY_H
+#include <cmath>
#include <cstdint>
#include <memory>
+#include <string>
+#include <vector>
+#include "arrow/buffer.h"
#include "arrow/type.h"
#include "arrow/util/bit-util.h"
#include "arrow/util/macros.h"
@@ -28,7 +32,6 @@
namespace arrow {
-class Buffer;
class MemoryPool;
class MutableBuffer;
class Status;
@@ -110,6 +113,374 @@ typedef std::shared_ptr<Array> ArrayPtr;
Status ARROW_EXPORT GetEmptyBitmap(
MemoryPool* pool, int32_t length, std::shared_ptr<MutableBuffer>* result);
+// Base class for fixed-size logical types
+class ARROW_EXPORT PrimitiveArray : public Array {
+ public:
+ virtual ~PrimitiveArray() {}
+
+ std::shared_ptr<Buffer> data() const { return data_; }
+
+ bool EqualsExact(const PrimitiveArray& other) const;
+ bool Equals(const std::shared_ptr<Array>& arr) const override;
+
+ protected:
+ PrimitiveArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+ int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+ std::shared_ptr<Buffer> data_;
+ const uint8_t* raw_data_;
+};
+
+template <typename TYPE>
+class ARROW_EXPORT NumericArray : public PrimitiveArray {
+ public:
+ using TypeClass = TYPE;
+ using value_type = typename TypeClass::c_type;
+ NumericArray(int32_t length, const std::shared_ptr<Buffer>& data,
+ int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+ : PrimitiveArray(
+ std::make_shared<TypeClass>(), length, data, null_count, null_bitmap) {}
+ NumericArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+ int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr)
+ : PrimitiveArray(type, length, data, null_count, null_bitmap) {}
+
+ bool EqualsExact(const NumericArray<TypeClass>& other) const {
+ return PrimitiveArray::EqualsExact(static_cast<const PrimitiveArray&>(other));
+ }
+
+ bool ApproxEquals(const std::shared_ptr<Array>& arr) const { return Equals(arr); }
+
+ bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const ArrayPtr& arr) const override {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+ const auto other = static_cast<NumericArray<TypeClass>*>(arr.get());
+ for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
+ const bool is_null = IsNull(i);
+ if (is_null != arr->IsNull(o_i) || (!is_null && Value(i) != other->Value(o_i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ const value_type* raw_data() const {
+ return reinterpret_cast<const value_type*>(raw_data_);
+ }
+
+ Status Accept(ArrayVisitor* visitor) const override;
+
+ value_type Value(int i) const { return raw_data()[i]; }
+};
+
+template <>
+inline bool NumericArray<FloatType>::ApproxEquals(
+ const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+
+ const auto& other = *static_cast<NumericArray<FloatType>*>(arr.get());
+
+ if (this == &other) { return true; }
+ if (null_count_ != other.null_count_) { return false; }
+
+ auto this_data = reinterpret_cast<const float*>(raw_data_);
+ auto other_data = reinterpret_cast<const float*>(other.raw_data_);
+
+ static constexpr float EPSILON = 1E-5;
+
+ if (length_ == 0 && other.length_ == 0) { return true; }
+
+ if (null_count_ > 0) {
+ bool equal_bitmap =
+ null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+ if (!equal_bitmap) { return false; }
+
+ for (int i = 0; i < length_; ++i) {
+ if (IsNull(i)) continue;
+ if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+ }
+ } else {
+ for (int i = 0; i < length_; ++i) {
+ if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+ }
+ }
+ return true;
+}
+
+template <>
+inline bool NumericArray<DoubleType>::ApproxEquals(
+ const std::shared_ptr<Array>& arr) const {
+ if (this == arr.get()) { return true; }
+ if (!arr) { return false; }
+ if (this->type_enum() != arr->type_enum()) { return false; }
+
+ const auto& other = *static_cast<NumericArray<DoubleType>*>(arr.get());
+
+ if (this == &other) { return true; }
+ if (null_count_ != other.null_count_) { return false; }
+
+ auto this_data = reinterpret_cast<const double*>(raw_data_);
+ auto other_data = reinterpret_cast<const double*>(other.raw_data_);
+
+ if (length_ == 0 && other.length_ == 0) { return true; }
+
+ static constexpr double EPSILON = 1E-5;
+
+ if (null_count_ > 0) {
+ bool equal_bitmap =
+ null_bitmap_->Equals(*other.null_bitmap_, BitUtil::CeilByte(length_) / 8);
+ if (!equal_bitmap) { return false; }
+
+ for (int i = 0; i < length_; ++i) {
+ if (IsNull(i)) continue;
+ if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+ }
+ } else {
+ for (int i = 0; i < length_; ++i) {
+ if (fabs(this_data[i] - other_data[i]) > EPSILON) { return false; }
+ }
+ }
+ return true;
+}
+
+class ARROW_EXPORT BooleanArray : public PrimitiveArray {
+ public:
+ using TypeClass = BooleanType;
+
+ BooleanArray(int32_t length, const std::shared_ptr<Buffer>& data,
+ int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+ BooleanArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& data,
+ int32_t null_count = 0, const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+ bool EqualsExact(const BooleanArray& other) const;
+ bool Equals(const ArrayPtr& arr) const override;
+ bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const ArrayPtr& arr) const override;
+
+ Status Accept(ArrayVisitor* visitor) const override;
+
+ const uint8_t* raw_data() const { return reinterpret_cast<const uint8_t*>(raw_data_); }
+
+ bool Value(int i) const { return BitUtil::GetBit(raw_data(), i); }
+};
+
+// ----------------------------------------------------------------------
+// ListArray
+
+class ARROW_EXPORT ListArray : public Array {
+ public:
+ using TypeClass = ListType;
+
+ ListArray(const TypePtr& type, int32_t length, std::shared_ptr<Buffer> offsets,
+ const ArrayPtr& values, int32_t null_count = 0,
+ std::shared_ptr<Buffer> null_bitmap = nullptr)
+ : Array(type, length, null_count, null_bitmap) {
+ offset_buffer_ = offsets;
+ offsets_ = offsets == nullptr ? nullptr : reinterpret_cast<const int32_t*>(
+ offset_buffer_->data());
+ values_ = values;
+ }
+
+ Status Validate() const override;
+
+ virtual ~ListArray() = default;
+
+ // Return a shared pointer in case the requestor desires to share ownership
+ // with this array.
+ std::shared_ptr<Array> values() const { return values_; }
+ std::shared_ptr<Buffer> offsets() const {
+ return std::static_pointer_cast<Buffer>(offset_buffer_);
+ }
+
+ std::shared_ptr<DataType> value_type() const { return values_->type(); }
+
+ const int32_t* raw_offsets() const { return offsets_; }
+
+ int32_t offset(int i) const { return offsets_[i]; }
+
+ // Neither of these functions will perform boundschecking
+ int32_t value_offset(int i) const { return offsets_[i]; }
+ int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+
+ bool EqualsExact(const ListArray& other) const;
+ bool Equals(const std::shared_ptr<Array>& arr) const override;
+
+ bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const ArrayPtr& arr) const override;
+
+ Status Accept(ArrayVisitor* visitor) const override;
+
+ protected:
+ std::shared_ptr<Buffer> offset_buffer_;
+ const int32_t* offsets_;
+ ArrayPtr values_;
+};
+
+// ----------------------------------------------------------------------
+// Binary and String
+
+class ARROW_EXPORT BinaryArray : public Array {
+ public:
+ using TypeClass = BinaryType;
+
+ BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+ const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+ const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+ // Constructor that allows sub-classes/builders to propagate there logical type up the
+ // class hierarchy.
+ BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
+ const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+ const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+ // Return the pointer to the given elements bytes
+ // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
+ // pointer + offset
+ const uint8_t* GetValue(int i, int32_t* out_length) const {
+ const int32_t pos = offsets_[i];
+ *out_length = offsets_[i + 1] - pos;
+ return data_ + pos;
+ }
+
+ std::shared_ptr<Buffer> data() const { return data_buffer_; }
+ std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
+
+ const int32_t* raw_offsets() const { return offsets_; }
+
+ int32_t offset(int i) const { return offsets_[i]; }
+
+ // Neither of these functions will perform boundschecking
+ int32_t value_offset(int i) const { return offsets_[i]; }
+ int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
+
+ bool EqualsExact(const BinaryArray& other) const;
+ bool Equals(const std::shared_ptr<Array>& arr) const override;
+ bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const ArrayPtr& arr) const override;
+
+ Status Validate() const override;
+
+ Status Accept(ArrayVisitor* visitor) const override;
+
+ private:
+ std::shared_ptr<Buffer> offset_buffer_;
+ const int32_t* offsets_;
+
+ std::shared_ptr<Buffer> data_buffer_;
+ const uint8_t* data_;
+};
+
+class ARROW_EXPORT StringArray : public BinaryArray {
+ public:
+ using TypeClass = StringType;
+
+ StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
+ const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
+ const std::shared_ptr<Buffer>& null_bitmap = nullptr);
+
+ // Construct a std::string
+ // TODO: std::bad_alloc possibility
+ std::string GetString(int i) const {
+ int32_t nchars;
+ const uint8_t* str = GetValue(i, &nchars);
+ return std::string(reinterpret_cast<const char*>(str), nchars);
+ }
+
+ Status Validate() const override;
+
+ Status Accept(ArrayVisitor* visitor) const override;
+};
+
+// ----------------------------------------------------------------------
+// Struct
+
+class ARROW_EXPORT StructArray : public Array {
+ public:
+ using TypeClass = StructType;
+
+ StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
+ int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
+ : Array(type, length, null_count, null_bitmap) {
+ type_ = type;
+ field_arrays_ = field_arrays;
+ }
+
+ Status Validate() const override;
+
+ virtual ~StructArray() {}
+
+ // Return a shared pointer in case the requestor desires to share ownership
+ // with this array.
+ std::shared_ptr<Array> field(int32_t pos) const;
+
+ const std::vector<ArrayPtr>& fields() const { return field_arrays_; }
+
+ bool EqualsExact(const StructArray& other) const;
+ bool Equals(const std::shared_ptr<Array>& arr) const override;
+ bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
+ const std::shared_ptr<Array>& arr) const override;
+
+ Status Accept(ArrayVisitor* visitor) const override;
+
+ protected:
+ // The child arrays corresponding to each field of the struct data type.
+ std::vector<ArrayPtr> field_arrays_;
+};
+
+// ----------------------------------------------------------------------
+// Union
+
+class UnionArray : public Array {
+ protected:
+ // The data are types encoded as int16
+ Buffer* types_;
+ std::vector<std::shared_ptr<Array>> children_;
+};
+
+class DenseUnionArray : public UnionArray {
+ protected:
+ Buffer* offset_buf_;
+};
+
+class SparseUnionArray : public UnionArray {};
+
+// ----------------------------------------------------------------------
+// extern templates and other details
+
+// gcc and clang disagree about how to handle template visibility when you have
+// explicit specializations https://llvm.org/bugs/show_bug.cgi?id=24815
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wattributes"
+#endif
+
+// Only instantiate these templates once
+extern template class ARROW_EXPORT NumericArray<Int8Type>;
+extern template class ARROW_EXPORT NumericArray<UInt8Type>;
+extern template class ARROW_EXPORT NumericArray<Int16Type>;
+extern template class ARROW_EXPORT NumericArray<UInt16Type>;
+extern template class ARROW_EXPORT NumericArray<Int32Type>;
+extern template class ARROW_EXPORT NumericArray<UInt32Type>;
+extern template class ARROW_EXPORT NumericArray<Int64Type>;
+extern template class ARROW_EXPORT NumericArray<UInt64Type>;
+extern template class ARROW_EXPORT NumericArray<HalfFloatType>;
+extern template class ARROW_EXPORT NumericArray<FloatType>;
+extern template class ARROW_EXPORT NumericArray<DoubleType>;
+extern template class ARROW_EXPORT NumericArray<TimestampType>;
+
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+// ----------------------------------------------------------------------
+// Helper functions
+
+// Create new arrays for logical types that are backed by primitive arrays.
+Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr<DataType>& type,
+ int32_t length, const std::shared_ptr<Buffer>& data, int32_t null_count,
+ const std::shared_ptr<Buffer>& null_bitmap, std::shared_ptr<Array>* out);
+
} // namespace arrow
#endif
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/buffer-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/buffer-test.cc b/cpp/src/arrow/buffer-test.cc
new file mode 100644
index 0000000..c1d027b
--- /dev/null
+++ b/cpp/src/arrow/buffer-test.cc
@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <limits>
+#include <string>
+
+#include "gtest/gtest.h"
+
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+
+using std::string;
+
+namespace arrow {
+
+class TestBuffer : public ::testing::Test {};
+
+TEST_F(TestBuffer, IsMutableFlag) {
+ Buffer buf(nullptr, 0);
+
+ ASSERT_FALSE(buf.is_mutable());
+
+ MutableBuffer mbuf(nullptr, 0);
+ ASSERT_TRUE(mbuf.is_mutable());
+
+ PoolBuffer pbuf;
+ ASSERT_TRUE(pbuf.is_mutable());
+}
+
+TEST_F(TestBuffer, Resize) {
+ PoolBuffer buf;
+
+ ASSERT_EQ(0, buf.size());
+ ASSERT_OK(buf.Resize(100));
+ ASSERT_EQ(100, buf.size());
+ ASSERT_OK(buf.Resize(200));
+ ASSERT_EQ(200, buf.size());
+
+ // Make it smaller, too
+ ASSERT_OK(buf.Resize(50));
+ ASSERT_EQ(50, buf.size());
+}
+
+TEST_F(TestBuffer, ResizeOOM) {
+ // realloc fails, even though there may be no explicit limit
+ PoolBuffer buf;
+ ASSERT_OK(buf.Resize(100));
+ int64_t to_alloc = std::numeric_limits<int64_t>::max();
+ ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
+}
+
+TEST_F(TestBuffer, EqualsWithSameContent) {
+ MemoryPool* pool = default_memory_pool();
+ const int32_t bufferSize = 128 * 1024;
+ uint8_t* rawBuffer1;
+ ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer1));
+ memset(rawBuffer1, 12, bufferSize);
+ uint8_t* rawBuffer2;
+ ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer2));
+ memset(rawBuffer2, 12, bufferSize);
+ uint8_t* rawBuffer3;
+ ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer3));
+ memset(rawBuffer3, 3, bufferSize);
+
+ Buffer buffer1(rawBuffer1, bufferSize);
+ Buffer buffer2(rawBuffer2, bufferSize);
+ Buffer buffer3(rawBuffer3, bufferSize);
+ ASSERT_TRUE(buffer1.Equals(buffer2));
+ ASSERT_FALSE(buffer1.Equals(buffer3));
+
+ pool->Free(rawBuffer1, bufferSize);
+ pool->Free(rawBuffer2, bufferSize);
+ pool->Free(rawBuffer3, bufferSize);
+}
+
+TEST_F(TestBuffer, EqualsWithSameBuffer) {
+ MemoryPool* pool = default_memory_pool();
+ const int32_t bufferSize = 128 * 1024;
+ uint8_t* rawBuffer;
+ ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer));
+ memset(rawBuffer, 111, bufferSize);
+
+ Buffer buffer1(rawBuffer, bufferSize);
+ Buffer buffer2(rawBuffer, bufferSize);
+ ASSERT_TRUE(buffer1.Equals(buffer2));
+
+ const int64_t nbytes = bufferSize / 2;
+ Buffer buffer3(rawBuffer, nbytes);
+ ASSERT_TRUE(buffer1.Equals(buffer3, nbytes));
+ ASSERT_FALSE(buffer1.Equals(buffer3, nbytes + 1));
+
+ pool->Free(rawBuffer, bufferSize);
+}
+
+TEST_F(TestBuffer, Copy) {
+ std::string data_str = "some data to copy";
+
+ auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
+
+ Buffer buf(data, data_str.size());
+
+ std::shared_ptr<Buffer> out;
+
+ ASSERT_OK(buf.Copy(5, 4, &out));
+
+ Buffer expected(data + 5, 4);
+ ASSERT_TRUE(out->Equals(expected));
+}
+
+TEST_F(TestBuffer, SliceBuffer) {
+ std::string data_str = "some data to slice";
+
+ auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
+
+ auto buf = std::make_shared<Buffer>(data, data_str.size());
+
+ std::shared_ptr<Buffer> out = SliceBuffer(buf, 5, 4);
+ Buffer expected(data + 5, 4);
+ ASSERT_TRUE(out->Equals(expected));
+
+ ASSERT_EQ(2, buf.use_count());
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/buffer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc
new file mode 100644
index 0000000..6ffa03a
--- /dev/null
+++ b/cpp/src/arrow/buffer.cc
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/buffer.h"
+
+#include <cstdint>
+#include <limits>
+
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
+#include "arrow/util/bit-util.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) {
+ data_ = parent->data() + offset;
+ size_ = size;
+ parent_ = parent;
+ capacity_ = size;
+}
+
+Buffer::~Buffer() {}
+
+Status Buffer::Copy(
+ int64_t start, int64_t nbytes, MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
+ // Sanity checks
+ DCHECK_LT(start, size_);
+ DCHECK_LE(nbytes, size_ - start);
+
+ auto new_buffer = std::make_shared<PoolBuffer>(pool);
+ RETURN_NOT_OK(new_buffer->Resize(nbytes));
+
+ std::memcpy(new_buffer->mutable_data(), data() + start, nbytes);
+
+ *out = new_buffer;
+ return Status::OK();
+}
+
+Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const {
+ return Copy(start, nbytes, default_memory_pool(), out);
+}
+
+std::shared_ptr<Buffer> SliceBuffer(
+ const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
+ DCHECK_LT(offset, buffer->size());
+ DCHECK_LE(length, buffer->size() - offset);
+ return std::make_shared<Buffer>(buffer, offset, length);
+}
+
+std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
+ return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
+}
+
+PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
+ if (pool == nullptr) { pool = default_memory_pool(); }
+ pool_ = pool;
+}
+
+PoolBuffer::~PoolBuffer() {
+ if (mutable_data_ != nullptr) { pool_->Free(mutable_data_, capacity_); }
+}
+
+Status PoolBuffer::Reserve(int64_t new_capacity) {
+ if (!mutable_data_ || new_capacity > capacity_) {
+ uint8_t* new_data;
+ new_capacity = BitUtil::RoundUpToMultipleOf64(new_capacity);
+ if (mutable_data_) {
+ RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
+ memcpy(new_data, mutable_data_, size_);
+ pool_->Free(mutable_data_, capacity_);
+ } else {
+ RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
+ }
+ mutable_data_ = new_data;
+ data_ = mutable_data_;
+ capacity_ = new_capacity;
+ }
+ return Status::OK();
+}
+
+Status PoolBuffer::Resize(int64_t new_size) {
+ RETURN_NOT_OK(Reserve(new_size));
+ size_ = new_size;
+ return Status::OK();
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/buffer.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h
new file mode 100644
index 0000000..27437ca
--- /dev/null
+++ b/cpp/src/arrow/buffer.h
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_BUFFER_H
+#define ARROW_UTIL_BUFFER_H
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+
+#include "arrow/status.h"
+#include "arrow/util/macros.h"
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class MemoryPool;
+class Status;
+
+// ----------------------------------------------------------------------
+// Buffer classes
+
+// Immutable API for a chunk of bytes which may or may not be owned by the
+// class instance. Buffers have two related notions of length: size and
+// capacity. Size is the number of bytes that might have valid data.
+// Capacity is the number of bytes that where allocated for the buffer in
+// total.
+// The following invariant is always true: Size < Capacity
+class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
+ public:
+ Buffer(const uint8_t* data, int64_t size)
+ : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
+ virtual ~Buffer();
+
+ // An offset into data that is owned by another buffer, but we want to be
+ // able to retain a valid pointer to it even after other shared_ptr's to the
+ // parent buffer have been destroyed
+ //
+ // This method makes no assertions about alignment or padding of the buffer but
+ // in general we expected buffers to be aligned and padded to 64 bytes. In the future
+ // we might add utility methods to help determine if a buffer satisfies this contract.
+ Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
+
+ std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
+
+ bool is_mutable() const { return is_mutable_; }
+
+ // Return true if both buffers are the same size and contain the same bytes
+ // up to the number of compared bytes
+ bool Equals(const Buffer& other, int64_t nbytes) const {
+ return this == &other ||
+ (size_ >= nbytes && other.size_ >= nbytes &&
+ (data_ == other.data_ || !memcmp(data_, other.data_, nbytes)));
+ }
+
+ bool Equals(const Buffer& other) const {
+ return this == &other ||
+ (size_ == other.size_ &&
+ (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
+ }
+
+ // Copy section of buffer into a new Buffer
+ Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
+ std::shared_ptr<Buffer>* out) const;
+
+ // Default memory pool
+ Status Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const;
+
+ int64_t capacity() const { return capacity_; }
+ const uint8_t* data() const { return data_; }
+
+ int64_t size() const { return size_; }
+
+ std::shared_ptr<Buffer> parent() const { return parent_; }
+
+ protected:
+ bool is_mutable_;
+ const uint8_t* data_;
+ int64_t size_;
+ int64_t capacity_;
+
+ // nullptr by default, but may be set
+ std::shared_ptr<Buffer> parent_;
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(Buffer);
+};
+
+// Construct a view on passed buffer at the indicated offset and length. This
+// function cannot fail and does not error checking (except in debug builds)
+ARROW_EXPORT std::shared_ptr<Buffer> SliceBuffer(
+ const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
+
+// A Buffer whose contents can be mutated. May or may not own its data.
+class ARROW_EXPORT MutableBuffer : public Buffer {
+ public:
+ MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
+ is_mutable_ = true;
+ mutable_data_ = data;
+ }
+
+ uint8_t* mutable_data() { return mutable_data_; }
+
+ // Get a read-only view of this buffer
+ std::shared_ptr<Buffer> GetImmutableView();
+
+ protected:
+ MutableBuffer() : Buffer(nullptr, 0), mutable_data_(nullptr) {}
+
+ uint8_t* mutable_data_;
+};
+
+class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
+ public:
+ // Change buffer reported size to indicated size, allocating memory if
+ // necessary. This will ensure that the capacity of the buffer is a multiple
+ // of 64 bytes as defined in Layout.md.
+ virtual Status Resize(int64_t new_size) = 0;
+
+ // Ensure that buffer has enough memory allocated to fit the indicated
+ // capacity (and meets the 64 byte padding requirement in Layout.md).
+ // It does not change buffer's reported size.
+ virtual Status Reserve(int64_t new_capacity) = 0;
+
+ protected:
+ ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
+};
+
+// A Buffer whose lifetime is tied to a particular MemoryPool
+class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
+ public:
+ explicit PoolBuffer(MemoryPool* pool = nullptr);
+ virtual ~PoolBuffer();
+
+ Status Resize(int64_t new_size) override;
+ Status Reserve(int64_t new_capacity) override;
+
+ private:
+ MemoryPool* pool_;
+};
+
+class ARROW_EXPORT BufferBuilder {
+ public:
+ explicit BufferBuilder(MemoryPool* pool)
+ : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
+
+ // Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
+ Status Resize(int32_t elements) {
+ if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
+ RETURN_NOT_OK(buffer_->Resize(elements));
+ capacity_ = buffer_->capacity();
+ data_ = buffer_->mutable_data();
+ return Status::OK();
+ }
+
+ Status Append(const uint8_t* data, int length) {
+ if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
+ UnsafeAppend(data, length);
+ return Status::OK();
+ }
+
+ template <typename T>
+ Status Append(T arithmetic_value) {
+ static_assert(std::is_arithmetic<T>::value,
+ "Convenience buffer append only supports arithmetic types");
+ return Append(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+ }
+
+ template <typename T>
+ Status Append(const T* arithmetic_values, int num_elements) {
+ static_assert(std::is_arithmetic<T>::value,
+ "Convenience buffer append only supports arithmetic types");
+ return Append(
+ reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
+ }
+
+ // Unsafe methods don't check existing size
+ void UnsafeAppend(const uint8_t* data, int length) {
+ memcpy(data_ + size_, data, length);
+ size_ += length;
+ }
+
+ template <typename T>
+ void UnsafeAppend(T arithmetic_value) {
+ static_assert(std::is_arithmetic<T>::value,
+ "Convenience buffer append only supports arithmetic types");
+ UnsafeAppend(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
+ }
+
+ template <typename T>
+ void UnsafeAppend(const T* arithmetic_values, int num_elements) {
+ static_assert(std::is_arithmetic<T>::value,
+ "Convenience buffer append only supports arithmetic types");
+ UnsafeAppend(
+ reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
+ }
+
+ std::shared_ptr<Buffer> Finish() {
+ auto result = buffer_;
+ buffer_ = nullptr;
+ capacity_ = size_ = 0;
+ return result;
+ }
+ int capacity() { return capacity_; }
+ int length() { return size_; }
+
+ private:
+ std::shared_ptr<PoolBuffer> buffer_;
+ MemoryPool* pool_;
+ uint8_t* data_;
+ int64_t capacity_;
+ int64_t size_;
+};
+
+} // namespace arrow
+
+#endif // ARROW_UTIL_BUFFER_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/builder.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 151b257..493b5e7 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -17,11 +17,17 @@
#include "arrow/builder.h"
+#include <cstdint>
#include <cstring>
+#include <limits>
+#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/status.h"
+#include "arrow/type.h"
+#include "arrow/type_traits.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/util/logging.h"
namespace arrow {
@@ -123,4 +129,323 @@ void ArrayBuilder::UnsafeSetNotNull(int32_t length) {
length_ = new_length;
}
+template <typename T>
+Status PrimitiveBuilder<T>::Init(int32_t capacity) {
+ RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+ data_ = std::make_shared<PoolBuffer>(pool_);
+
+ int64_t nbytes = TypeTraits<T>::bytes_required(capacity);
+ RETURN_NOT_OK(data_->Resize(nbytes));
+ // TODO(emkornfield) valgrind complains without this
+ memset(data_->mutable_data(), 0, nbytes);
+
+ raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+ return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Resize(int32_t capacity) {
+ // XXX: Set floor size for now
+ if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+ if (capacity_ == 0) {
+ RETURN_NOT_OK(Init(capacity));
+ } else {
+ RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+ const int64_t old_bytes = data_->size();
+ const int64_t new_bytes = TypeTraits<T>::bytes_required(capacity);
+ RETURN_NOT_OK(data_->Resize(new_bytes));
+ raw_data_ = reinterpret_cast<value_type*>(data_->mutable_data());
+ memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+ }
+ return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Append(
+ const value_type* values, int32_t length, const uint8_t* valid_bytes) {
+ RETURN_NOT_OK(Reserve(length));
+
+ if (length > 0) {
+ memcpy(raw_data_ + length_, values, TypeTraits<T>::bytes_required(length));
+ }
+
+ // length_ is update by these
+ ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
+
+ return Status::OK();
+}
+
+template <typename T>
+Status PrimitiveBuilder<T>::Finish(std::shared_ptr<Array>* out) {
+ const int64_t bytes_required = TypeTraits<T>::bytes_required(length_);
+ if (bytes_required > 0 && bytes_required < data_->size()) {
+ // Trim buffers
+ RETURN_NOT_OK(data_->Resize(bytes_required));
+ }
+ *out = std::make_shared<typename TypeTraits<T>::ArrayType>(
+ type_, length_, data_, null_count_, null_bitmap_);
+
+ data_ = null_bitmap_ = nullptr;
+ capacity_ = length_ = null_count_ = 0;
+ return Status::OK();
+}
+
+template class PrimitiveBuilder<UInt8Type>;
+template class PrimitiveBuilder<UInt16Type>;
+template class PrimitiveBuilder<UInt32Type>;
+template class PrimitiveBuilder<UInt64Type>;
+template class PrimitiveBuilder<Int8Type>;
+template class PrimitiveBuilder<Int16Type>;
+template class PrimitiveBuilder<Int32Type>;
+template class PrimitiveBuilder<Int64Type>;
+template class PrimitiveBuilder<TimestampType>;
+template class PrimitiveBuilder<HalfFloatType>;
+template class PrimitiveBuilder<FloatType>;
+template class PrimitiveBuilder<DoubleType>;
+
+Status BooleanBuilder::Init(int32_t capacity) {
+ RETURN_NOT_OK(ArrayBuilder::Init(capacity));
+ data_ = std::make_shared<PoolBuffer>(pool_);
+
+ int64_t nbytes = BitUtil::BytesForBits(capacity);
+ RETURN_NOT_OK(data_->Resize(nbytes));
+ // TODO(emkornfield) valgrind complains without this
+ memset(data_->mutable_data(), 0, nbytes);
+
+ raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+ return Status::OK();
+}
+
+Status BooleanBuilder::Resize(int32_t capacity) {
+ // XXX: Set floor size for now
+ if (capacity < kMinBuilderCapacity) { capacity = kMinBuilderCapacity; }
+
+ if (capacity_ == 0) {
+ RETURN_NOT_OK(Init(capacity));
+ } else {
+ RETURN_NOT_OK(ArrayBuilder::Resize(capacity));
+ const int64_t old_bytes = data_->size();
+ const int64_t new_bytes = BitUtil::BytesForBits(capacity);
+
+ RETURN_NOT_OK(data_->Resize(new_bytes));
+ raw_data_ = reinterpret_cast<uint8_t*>(data_->mutable_data());
+ memset(data_->mutable_data() + old_bytes, 0, new_bytes - old_bytes);
+ }
+ return Status::OK();
+}
+
+Status BooleanBuilder::Finish(std::shared_ptr<Array>* out) {
+ const int64_t bytes_required = BitUtil::BytesForBits(length_);
+
+ if (bytes_required > 0 && bytes_required < data_->size()) {
+ // Trim buffers
+ RETURN_NOT_OK(data_->Resize(bytes_required));
+ }
+ *out = std::make_shared<BooleanArray>(type_, length_, data_, null_count_, null_bitmap_);
+
+ data_ = null_bitmap_ = nullptr;
+ capacity_ = length_ = null_count_ = 0;
+ return Status::OK();
+}
+
+Status BooleanBuilder::Append(
+ const uint8_t* values, int32_t length, const uint8_t* valid_bytes) {
+ RETURN_NOT_OK(Reserve(length));
+
+ for (int i = 0; i < length; ++i) {
+ // Skip reading from unitialised memory
+ // TODO: This actually is only to keep valgrind happy but may or may not
+ // have a performance impact.
+ if ((valid_bytes != nullptr) && !valid_bytes[i]) continue;
+
+ if (values[i] > 0) {
+ BitUtil::SetBit(raw_data_, length_ + i);
+ } else {
+ BitUtil::ClearBit(raw_data_, length_ + i);
+ }
+ }
+
+ // this updates length_
+ ArrayBuilder::UnsafeAppendToBitmap(valid_bytes, length);
+ return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// ListBuilder
+
+ListBuilder::ListBuilder(
+ MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder, const TypePtr& type)
+ : ArrayBuilder(
+ pool, type ? type : std::static_pointer_cast<DataType>(
+ std::make_shared<ListType>(value_builder->type()))),
+ offset_builder_(pool),
+ value_builder_(value_builder) {}
+
+ListBuilder::ListBuilder(
+ MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type)
+ : ArrayBuilder(pool, type ? type : std::static_pointer_cast<DataType>(
+ std::make_shared<ListType>(values->type()))),
+ offset_builder_(pool),
+ values_(values) {}
+
+Status ListBuilder::Init(int32_t elements) {
+ DCHECK_LT(elements, std::numeric_limits<int32_t>::max());
+ RETURN_NOT_OK(ArrayBuilder::Init(elements));
+ // one more then requested for offsets
+ return offset_builder_.Resize((elements + 1) * sizeof(int32_t));
+}
+
+Status ListBuilder::Resize(int32_t capacity) {
+ DCHECK_LT(capacity, std::numeric_limits<int32_t>::max());
+ // one more then requested for offsets
+ RETURN_NOT_OK(offset_builder_.Resize((capacity + 1) * sizeof(int32_t)));
+ return ArrayBuilder::Resize(capacity);
+}
+
+Status ListBuilder::Finish(std::shared_ptr<Array>* out) {
+ std::shared_ptr<Array> items = values_;
+ if (!items) { RETURN_NOT_OK(value_builder_->Finish(&items)); }
+
+ RETURN_NOT_OK(offset_builder_.Append<int32_t>(items->length()));
+ std::shared_ptr<Buffer> offsets = offset_builder_.Finish();
+
+ *out = std::make_shared<ListArray>(
+ type_, length_, offsets, items, null_count_, null_bitmap_);
+
+ Reset();
+
+ return Status::OK();
+}
+
+void ListBuilder::Reset() {
+ capacity_ = length_ = null_count_ = 0;
+ null_bitmap_ = nullptr;
+}
+
+std::shared_ptr<ArrayBuilder> ListBuilder::value_builder() const {
+ DCHECK(!values_) << "Using value builder is pointless when values_ is set";
+ return value_builder_;
+}
+
+// ----------------------------------------------------------------------
+// String and binary
+
+// This used to be a static member variable of BinaryBuilder, but it can cause
+// valgrind to report a (spurious?) memory leak when needed in other shared
+// libraries. The problem came up while adding explicit visibility to libarrow
+// and libparquet_arrow
+static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
+
+BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
+ : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, kBinaryValueType), type) {
+ byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
+}
+
+Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
+ std::shared_ptr<Array> result;
+ RETURN_NOT_OK(ListBuilder::Finish(&result));
+
+ const auto list = std::dynamic_pointer_cast<ListArray>(result);
+ auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+
+ *out = std::make_shared<BinaryArray>(list->length(), list->offsets(), values->data(),
+ list->null_count(), list->null_bitmap());
+ return Status::OK();
+}
+
+Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
+ std::shared_ptr<Array> result;
+ RETURN_NOT_OK(ListBuilder::Finish(&result));
+
+ const auto list = std::dynamic_pointer_cast<ListArray>(result);
+ auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
+
+ *out = std::make_shared<StringArray>(list->length(), list->offsets(), values->data(),
+ list->null_count(), list->null_bitmap());
+ return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Struct
+
+Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
+ std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
+ for (size_t i = 0; i < field_builders_.size(); ++i) {
+ RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
+ }
+
+ *out = std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
+
+ null_bitmap_ = nullptr;
+ capacity_ = length_ = null_count_ = 0;
+
+ return Status::OK();
+}
+
+std::shared_ptr<ArrayBuilder> StructBuilder::field_builder(int pos) const {
+ DCHECK_GT(field_builders_.size(), 0);
+ return field_builders_[pos];
+}
+
+// ----------------------------------------------------------------------
+// Helper functions
+
+#define BUILDER_CASE(ENUM, BuilderType) \
+ case Type::ENUM: \
+ out->reset(new BuilderType(pool, type)); \
+ return Status::OK();
+
+// Initially looked at doing this with vtables, but shared pointers makes it
+// difficult
+//
+// TODO(wesm): come up with a less monolithic strategy
+Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+ std::shared_ptr<ArrayBuilder>* out) {
+ switch (type->type) {
+ BUILDER_CASE(UINT8, UInt8Builder);
+ BUILDER_CASE(INT8, Int8Builder);
+ BUILDER_CASE(UINT16, UInt16Builder);
+ BUILDER_CASE(INT16, Int16Builder);
+ BUILDER_CASE(UINT32, UInt32Builder);
+ BUILDER_CASE(INT32, Int32Builder);
+ BUILDER_CASE(UINT64, UInt64Builder);
+ BUILDER_CASE(INT64, Int64Builder);
+ BUILDER_CASE(TIMESTAMP, TimestampBuilder);
+
+ BUILDER_CASE(BOOL, BooleanBuilder);
+
+ BUILDER_CASE(FLOAT, FloatBuilder);
+ BUILDER_CASE(DOUBLE, DoubleBuilder);
+
+ BUILDER_CASE(STRING, StringBuilder);
+ BUILDER_CASE(BINARY, BinaryBuilder);
+
+ case Type::LIST: {
+ std::shared_ptr<ArrayBuilder> value_builder;
+ std::shared_ptr<DataType> value_type =
+ static_cast<ListType*>(type.get())->value_type();
+ RETURN_NOT_OK(MakeBuilder(pool, value_type, &value_builder));
+ out->reset(new ListBuilder(pool, value_builder));
+ return Status::OK();
+ }
+
+ case Type::STRUCT: {
+ std::vector<FieldPtr>& fields = type->children_;
+ std::vector<std::shared_ptr<ArrayBuilder>> values_builder;
+
+ for (auto it : fields) {
+ std::shared_ptr<ArrayBuilder> builder;
+ RETURN_NOT_OK(MakeBuilder(pool, it->type, &builder));
+ values_builder.push_back(builder);
+ }
+ out->reset(new StructBuilder(pool, type, values_builder));
+ return Status::OK();
+ }
+
+ default:
+ return Status::NotImplemented(type->ToString());
+ }
+}
+
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/builder.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index 73e49c0..7162d31 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -20,18 +20,20 @@
#include <cstdint>
#include <memory>
+#include <string>
#include <vector>
+#include "arrow/buffer.h"
+#include "arrow/status.h"
#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
#include "arrow/util/macros.h"
-#include "arrow/util/status.h"
#include "arrow/util/visibility.h"
namespace arrow {
class Array;
class MemoryPool;
-class PoolBuffer;
static constexpr int32_t kMinBuilderCapacity = 1 << 5;
@@ -130,6 +132,315 @@ class ARROW_EXPORT ArrayBuilder {
DISALLOW_COPY_AND_ASSIGN(ArrayBuilder);
};
+template <typename Type>
+class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder {
+ public:
+ using value_type = typename Type::c_type;
+
+ explicit PrimitiveBuilder(MemoryPool* pool, const TypePtr& type)
+ : ArrayBuilder(pool, type), data_(nullptr) {}
+
+ virtual ~PrimitiveBuilder() {}
+
+ using ArrayBuilder::Advance;
+
+ // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+ Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+ RETURN_NOT_OK(Reserve(length));
+ UnsafeAppendToBitmap(valid_bytes, length);
+ return Status::OK();
+ }
+
+ Status AppendNull() {
+ RETURN_NOT_OK(Reserve(1));
+ UnsafeAppendToBitmap(false);
+ return Status::OK();
+ }
+
+ std::shared_ptr<Buffer> data() const { return data_; }
+
+ // Vector append
+ //
+ // If passed, valid_bytes is of equal length to values, and any zero byte
+ // will be considered as a null for that slot
+ Status Append(
+ const value_type* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+
+ Status Finish(std::shared_ptr<Array>* out) override;
+ Status Init(int32_t capacity) override;
+
+ // Increase the capacity of the builder to accommodate at least the indicated
+ // number of elements
+ Status Resize(int32_t capacity) override;
+
+ protected:
+ std::shared_ptr<PoolBuffer> data_;
+ value_type* raw_data_;
+};
+
+template <typename T>
+class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder<T> {
+ public:
+ using typename PrimitiveBuilder<T>::value_type;
+ using PrimitiveBuilder<T>::PrimitiveBuilder;
+
+ using PrimitiveBuilder<T>::Append;
+ using PrimitiveBuilder<T>::Init;
+ using PrimitiveBuilder<T>::Resize;
+ using PrimitiveBuilder<T>::Reserve;
+
+ // Scalar append.
+ Status Append(value_type val) {
+ RETURN_NOT_OK(ArrayBuilder::Reserve(1));
+ UnsafeAppend(val);
+ return Status::OK();
+ }
+
+ // Does not capacity-check; make sure to call Reserve beforehand
+ void UnsafeAppend(value_type val) {
+ BitUtil::SetBit(null_bitmap_data_, length_);
+ raw_data_[length_++] = val;
+ }
+
+ protected:
+ using PrimitiveBuilder<T>::length_;
+ using PrimitiveBuilder<T>::null_bitmap_data_;
+ using PrimitiveBuilder<T>::raw_data_;
+};
+
+// Builders
+
+using UInt8Builder = NumericBuilder<UInt8Type>;
+using UInt16Builder = NumericBuilder<UInt16Type>;
+using UInt32Builder = NumericBuilder<UInt32Type>;
+using UInt64Builder = NumericBuilder<UInt64Type>;
+
+using Int8Builder = NumericBuilder<Int8Type>;
+using Int16Builder = NumericBuilder<Int16Type>;
+using Int32Builder = NumericBuilder<Int32Type>;
+using Int64Builder = NumericBuilder<Int64Type>;
+using TimestampBuilder = NumericBuilder<TimestampType>;
+
+using HalfFloatBuilder = NumericBuilder<HalfFloatType>;
+using FloatBuilder = NumericBuilder<FloatType>;
+using DoubleBuilder = NumericBuilder<DoubleType>;
+
+class ARROW_EXPORT BooleanBuilder : public ArrayBuilder {
+ public:
+ explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type)
+ : ArrayBuilder(pool, type), data_(nullptr) {}
+
+ virtual ~BooleanBuilder() {}
+
+ using ArrayBuilder::Advance;
+
+ // Write nulls as uint8_t* (0 value indicates null) into pre-allocated memory
+ Status AppendNulls(const uint8_t* valid_bytes, int32_t length) {
+ RETURN_NOT_OK(Reserve(length));
+ UnsafeAppendToBitmap(valid_bytes, length);
+ return Status::OK();
+ }
+
+ Status AppendNull() {
+ RETURN_NOT_OK(Reserve(1));
+ UnsafeAppendToBitmap(false);
+ return Status::OK();
+ }
+
+ std::shared_ptr<Buffer> data() const { return data_; }
+
+ // Scalar append
+ Status Append(bool val) {
+ Reserve(1);
+ BitUtil::SetBit(null_bitmap_data_, length_);
+ if (val) {
+ BitUtil::SetBit(raw_data_, length_);
+ } else {
+ BitUtil::ClearBit(raw_data_, length_);
+ }
+ ++length_;
+ return Status::OK();
+ }
+
+ // Vector append
+ //
+ // If passed, valid_bytes is of equal length to values, and any zero byte
+ // will be considered as a null for that slot
+ Status Append(
+ const uint8_t* values, int32_t length, const uint8_t* valid_bytes = nullptr);
+
+ Status Finish(std::shared_ptr<Array>* out) override;
+ Status Init(int32_t capacity) override;
+
+ // Increase the capacity of the builder to accommodate at least the indicated
+ // number of elements
+ Status Resize(int32_t capacity) override;
+
+ protected:
+ std::shared_ptr<PoolBuffer> data_;
+ uint8_t* raw_data_;
+};
+
+// ----------------------------------------------------------------------
+// List builder
+
+// Builder class for variable-length list array value types
+//
+// To use this class, you must append values to the child array builder and use
+// the Append function to delimit each distinct list value (once the values
+// have been appended to the child array) or use the bulk API to append
+// a sequence of offests and null values.
+//
+// A note on types. Per arrow/type.h all types in the c++ implementation are
+// logical so even though this class always builds list array, this can
+// represent multiple different logical types. If no logical type is provided
+// at construction time, the class defaults to List<T> where t is taken from the
+// value_builder/values that the object is constructed with.
+class ARROW_EXPORT ListBuilder : public ArrayBuilder {
+ public:
+ // Use this constructor to incrementally build the value array along with offsets and
+ // null bitmap.
+ ListBuilder(MemoryPool* pool, std::shared_ptr<ArrayBuilder> value_builder,
+ const TypePtr& type = nullptr);
+
+ // Use this constructor to build the list with a pre-existing values array
+ ListBuilder(
+ MemoryPool* pool, std::shared_ptr<Array> values, const TypePtr& type = nullptr);
+
+ virtual ~ListBuilder() {}
+
+ Status Init(int32_t elements) override;
+ Status Resize(int32_t capacity) override;
+ Status Finish(std::shared_ptr<Array>* out) override;
+
+ // Vector append
+ //
+ // If passed, valid_bytes is of equal length to values, and any zero byte
+ // will be considered as a null for that slot
+ Status Append(
+ const int32_t* offsets, int32_t length, const uint8_t* valid_bytes = nullptr) {
+ RETURN_NOT_OK(Reserve(length));
+ UnsafeAppendToBitmap(valid_bytes, length);
+ offset_builder_.UnsafeAppend<int32_t>(offsets, length);
+ return Status::OK();
+ }
+
+ // Start a new variable-length list slot
+ //
+ // This function should be called before beginning to append elements to the
+ // value builder
+ Status Append(bool is_valid = true) {
+ RETURN_NOT_OK(Reserve(1));
+ UnsafeAppendToBitmap(is_valid);
+ RETURN_NOT_OK(offset_builder_.Append<int32_t>(value_builder_->length()));
+ return Status::OK();
+ }
+
+ Status AppendNull() { return Append(false); }
+
+ std::shared_ptr<ArrayBuilder> value_builder() const;
+
+ protected:
+ BufferBuilder offset_builder_;
+ std::shared_ptr<ArrayBuilder> value_builder_;
+ std::shared_ptr<Array> values_;
+
+ void Reset();
+};
+
+// ----------------------------------------------------------------------
+// Binary and String
+
+// BinaryBuilder : public ListBuilder
+class ARROW_EXPORT BinaryBuilder : public ListBuilder {
+ public:
+ explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
+ virtual ~BinaryBuilder() {}
+
+ Status Append(const uint8_t* value, int32_t length) {
+ RETURN_NOT_OK(ListBuilder::Append());
+ return byte_builder_->Append(value, length);
+ }
+
+ Status Append(const char* value, int32_t length) {
+ return Append(reinterpret_cast<const uint8_t*>(value), length);
+ }
+
+ Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
+
+ Status Finish(std::shared_ptr<Array>* out) override;
+
+ protected:
+ UInt8Builder* byte_builder_;
+};
+
+// String builder
+class ARROW_EXPORT StringBuilder : public BinaryBuilder {
+ public:
+ explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
+ : BinaryBuilder(pool, type) {}
+
+ using BinaryBuilder::Append;
+
+ Status Finish(std::shared_ptr<Array>* out) override;
+
+ Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
+};
+
+// ----------------------------------------------------------------------
+// Struct
+
+// ---------------------------------------------------------------------------------
+// StructArray builder
+// Append, Resize and Reserve methods are acting on StructBuilder.
+// Please make sure all these methods of all child-builders' are consistently
+// called to maintain data-structure consistency.
+class ARROW_EXPORT StructBuilder : public ArrayBuilder {
+ public:
+ StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+ const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
+ : ArrayBuilder(pool, type) {
+ field_builders_ = field_builders;
+ }
+
+ Status Finish(std::shared_ptr<Array>* out) override;
+
+ // Null bitmap is of equal length to every child field, and any zero byte
+ // will be considered as a null for that field, but users must using app-
+ // end methods or advance methods of the child builders' independently to
+ // insert data.
+ Status Append(int32_t length, const uint8_t* valid_bytes) {
+ RETURN_NOT_OK(Reserve(length));
+ UnsafeAppendToBitmap(valid_bytes, length);
+ return Status::OK();
+ }
+
+ // Append an element to the Struct. All child-builders' Append method must
+ // be called independently to maintain data-structure consistency.
+ Status Append(bool is_valid = true) {
+ RETURN_NOT_OK(Reserve(1));
+ UnsafeAppendToBitmap(is_valid);
+ return Status::OK();
+ }
+
+ Status AppendNull() { return Append(false); }
+
+ std::shared_ptr<ArrayBuilder> field_builder(int pos) const;
+
+ const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders() const {
+ return field_builders_;
+ }
+
+ protected:
+ std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
+};
+
+// ----------------------------------------------------------------------
+// Helper functions
+
+Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
+ std::shared_ptr<ArrayBuilder>* out);
+
} // namespace arrow
#endif // ARROW_BUILDER_H_
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/column-benchmark.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/column-benchmark.cc b/cpp/src/arrow/column-benchmark.cc
index f429a81..650ec90 100644
--- a/cpp/src/arrow/column-benchmark.cc
+++ b/cpp/src/arrow/column-benchmark.cc
@@ -17,9 +17,9 @@
#include "benchmark/benchmark.h"
+#include "arrow/array.h"
+#include "arrow/memory_pool.h"
#include "arrow/test-util.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/memory-pool.h"
namespace arrow {
namespace {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/column-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/column-test.cc b/cpp/src/arrow/column-test.cc
index ac3636d..9005245 100644
--- a/cpp/src/arrow/column-test.cc
+++ b/cpp/src/arrow/column-test.cc
@@ -27,7 +27,6 @@
#include "arrow/schema.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
-#include "arrow/types/primitive.h"
using std::shared_ptr;
using std::vector;
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/column.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/column.cc b/cpp/src/arrow/column.cc
index eca5f4d..1d136e7 100644
--- a/cpp/src/arrow/column.cc
+++ b/cpp/src/arrow/column.cc
@@ -21,8 +21,8 @@
#include <sstream>
#include "arrow/array.h"
+#include "arrow/status.h"
#include "arrow/type.h"
-#include "arrow/util/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/file.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc
index 05fa666..c50a9bb 100644
--- a/cpp/src/arrow/io/file.cc
+++ b/cpp/src/arrow/io/file.cc
@@ -107,9 +107,9 @@
#include "arrow/io/interfaces.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/hdfs.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 8c6d49f..b8e2120 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -22,10 +22,10 @@
#include <sstream>
#include <string>
+#include "arrow/buffer.h"
#include "arrow/io/hdfs.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/interfaces.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/interfaces.cc b/cpp/src/arrow/io/interfaces.cc
index 44986ce..68c1ac3 100644
--- a/cpp/src/arrow/io/interfaces.cc
+++ b/cpp/src/arrow/io/interfaces.cc
@@ -20,8 +20,8 @@
#include <cstdint>
#include <memory>
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/buffer.h"
+#include "arrow/status.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/io-file-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/io-file-test.cc b/cpp/src/arrow/io/io-file-test.cc
index fad49ce..f0ea7ec 100644
--- a/cpp/src/arrow/io/io-file-test.cc
+++ b/cpp/src/arrow/io/io-file-test.cc
@@ -30,7 +30,7 @@
#include "arrow/io/file.h"
#include "arrow/io/test-common.h"
-#include "arrow/util/memory-pool.h"
+#include "arrow/memory_pool.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/io-hdfs-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 8338de6..e07eaa3 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -25,8 +25,8 @@
#include <boost/filesystem.hpp> // NOLINT
#include "arrow/io/hdfs.h"
+#include "arrow/status.h"
#include "arrow/test-util.h"
-#include "arrow/util/status.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/libhdfs_shim.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc
index 36b8a4e..3715376 100644
--- a/cpp/src/arrow/io/libhdfs_shim.cc
+++ b/cpp/src/arrow/io/libhdfs_shim.cc
@@ -53,7 +53,7 @@ extern "C" {
#include <boost/filesystem.hpp> // NOLINT
-#include "arrow/util/status.h"
+#include "arrow/status.h"
#include "arrow/util/visibility.h"
namespace fs = boost::filesystem;
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/memory.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/memory.cc b/cpp/src/arrow/io/memory.cc
index af495e2..b5cf4b7 100644
--- a/cpp/src/arrow/io/memory.cc
+++ b/cpp/src/arrow/io/memory.cc
@@ -38,10 +38,9 @@
#include <sstream>
#include <string>
+#include "arrow/buffer.h"
#include "arrow/io/interfaces.h"
-
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/io/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index f8fed88..1468083 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -32,10 +32,10 @@
// nothing
#endif
+#include "arrow/buffer.h"
#include "arrow/io/memory.h"
+#include "arrow/memory_pool.h"
#include "arrow/test-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
namespace arrow {
namespace io {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/adapter.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
index edf716f..f813c1d 100644
--- a/cpp/src/arrow/ipc/adapter.cc
+++ b/cpp/src/arrow/ipc/adapter.cc
@@ -23,6 +23,7 @@
#include <vector>
#include "arrow/array.h"
+#include "arrow/buffer.h"
#include "arrow/io/interfaces.h"
#include "arrow/io/memory.h"
#include "arrow/ipc/Message_generated.h"
@@ -30,17 +31,11 @@
#include "arrow/ipc/metadata.h"
#include "arrow/ipc/util.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
#include "arrow/util/logging.h"
-#include "arrow/util/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/file.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/file.cc b/cpp/src/arrow/ipc/file.cc
index fa50058..d7d2e61 100644
--- a/cpp/src/arrow/ipc/file.cc
+++ b/cpp/src/arrow/ipc/file.cc
@@ -22,14 +22,14 @@
#include <sstream>
#include <vector>
+#include "arrow/buffer.h"
#include "arrow/io/interfaces.h"
#include "arrow/io/memory.h"
#include "arrow/ipc/adapter.h"
#include "arrow/ipc/metadata.h"
#include "arrow/ipc/util.h"
-#include "arrow/util/buffer.h"
+#include "arrow/status.h"
#include "arrow/util/logging.h"
-#include "arrow/util/status.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/ipc-adapter-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
index 1accfde..f309b85 100644
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ b/cpp/src/arrow/ipc/ipc-adapter-test.cc
@@ -30,15 +30,11 @@
#include "arrow/ipc/test-common.h"
#include "arrow/ipc/util.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
#include "arrow/test-util.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/ipc-file-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
index a1feac4..0a9f677 100644
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ b/cpp/src/arrow/ipc/ipc-file-test.cc
@@ -24,6 +24,7 @@
#include "gtest/gtest.h"
+#include "arrow/array.h"
#include "arrow/io/memory.h"
#include "arrow/io/test-common.h"
#include "arrow/ipc/adapter.h"
@@ -31,15 +32,11 @@
#include "arrow/ipc/test-common.h"
#include "arrow/ipc/util.h"
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
#include "arrow/test-util.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/ipc-json-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index ba4d9ca..f793a26 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -26,17 +26,15 @@
#include "gtest/gtest.h"
#include "arrow/array.h"
+#include "arrow/builder.h"
#include "arrow/ipc/json-internal.h"
#include "arrow/ipc/json.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
namespace arrow {
namespace ipc {
@@ -147,7 +145,7 @@ TEST(TestJsonArrayWriter, NestedTypes) {
std::vector<int32_t> values = {0, 1, 2, 3, 4, 5, 6};
std::shared_ptr<Array> values_array;
- MakeArray<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
+ ArrayFromVector<Int32Type, int32_t>(int32(), values_is_valid, values, &values_array);
// List
std::vector<bool> list_is_valid = {true, false, true, true, true};
@@ -188,10 +186,10 @@ void MakeBatchArrays(const std::shared_ptr<Schema>& schema, const int num_rows,
test::randint<int32_t>(num_rows, 0, 100, &v2_values);
std::shared_ptr<Array> v1;
- MakeArray<Int8Type, int8_t>(schema->field(0)->type, is_valid, v1_values, &v1);
+ ArrayFromVector<Int8Type, int8_t>(schema->field(0)->type, is_valid, v1_values, &v1);
std::shared_ptr<Array> v2;
- MakeArray<Int32Type, int32_t>(schema->field(1)->type, is_valid, v2_values, &v2);
+ ArrayFromVector<Int32Type, int32_t>(schema->field(1)->type, is_valid, v2_values, &v2);
static const int kBufferSize = 10;
static uint8_t buffer[kBufferSize];
@@ -323,13 +321,13 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
std::vector<bool> foo_valid = {true, false, true, true, true};
std::vector<int32_t> foo_values = {1, 2, 3, 4, 5};
std::shared_ptr<Array> foo;
- MakeArray<Int32Type, int32_t>(int32(), foo_valid, foo_values, &foo);
+ ArrayFromVector<Int32Type, int32_t>(int32(), foo_valid, foo_values, &foo);
ASSERT_TRUE(batch->column(0)->Equals(foo));
std::vector<bool> bar_valid = {true, false, false, true, true};
std::vector<double> bar_values = {1, 2, 3, 4, 5};
std::shared_ptr<Array> bar;
- MakeArray<DoubleType, double>(float64(), bar_valid, bar_values, &bar);
+ ArrayFromVector<DoubleType, double>(float64(), bar_valid, bar_values, &bar);
ASSERT_TRUE(batch->column(1)->Equals(bar));
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/ipc-metadata-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
index de08e6d..7c5744a 100644
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ b/cpp/src/arrow/ipc/ipc-metadata-test.cc
@@ -24,9 +24,9 @@
#include "arrow/io/memory.h"
#include "arrow/ipc/metadata.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
-#include "arrow/util/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/json-integration-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/json-integration-test.cc b/cpp/src/arrow/ipc/json-integration-test.cc
index 291a719..5e59356 100644
--- a/cpp/src/arrow/ipc/json-integration-test.cc
+++ b/cpp/src/arrow/ipc/json-integration-test.cc
@@ -33,9 +33,9 @@
#include "arrow/ipc/json.h"
#include "arrow/pretty_print.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/test-util.h"
-#include "arrow/util/status.h"
DEFINE_string(arrow, "", "Arrow file name");
DEFINE_string(json, "", "JSON file name");
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/json-internal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index ff9f598..db11b7d 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -28,16 +28,14 @@
#include "rapidjson/writer.h"
#include "arrow/array.h"
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/util/logging.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/json.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/json.cc b/cpp/src/arrow/ipc/json.cc
index 2281611..6e3a993 100644
--- a/cpp/src/arrow/ipc/json.cc
+++ b/cpp/src/arrow/ipc/json.cc
@@ -23,14 +23,14 @@
#include <vector>
#include "arrow/array.h"
+#include "arrow/buffer.h"
#include "arrow/ipc/json-internal.h"
+#include "arrow/memory_pool.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/type.h"
-#include "arrow/util/buffer.h"
#include "arrow/util/logging.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/metadata-internal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 5a27589..16069a8 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -25,11 +25,11 @@
#include "flatbuffers/flatbuffers.h"
+#include "arrow/buffer.h"
#include "arrow/ipc/Message_generated.h"
#include "arrow/schema.h"
+#include "arrow/status.h"
#include "arrow/type.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/metadata.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 44d3939..f0674ff 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -28,9 +28,9 @@
#include "arrow/ipc/Message_generated.h"
#include "arrow/ipc/metadata-internal.h"
+#include "arrow/buffer.h"
#include "arrow/schema.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 65b3782..8416f0d 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -25,16 +25,13 @@
#include <vector>
#include "arrow/array.h"
+#include "arrow/buffer.h"
+#include "arrow/builder.h"
+#include "arrow/memory_pool.h"
#include "arrow/table.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/memory-pool.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/ipc/util.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/util.h b/cpp/src/arrow/ipc/util.h
index 242d662..2000c61 100644
--- a/cpp/src/arrow/ipc/util.h
+++ b/cpp/src/arrow/ipc/util.h
@@ -22,7 +22,7 @@
#include "arrow/array.h"
#include "arrow/io/interfaces.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
namespace arrow {
namespace ipc {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/memory_pool-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
new file mode 100644
index 0000000..d6f323d
--- /dev/null
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <limits>
+
+#include "gtest/gtest.h"
+
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+TEST(DefaultMemoryPool, MemoryTracking) {
+ MemoryPool* pool = default_memory_pool();
+
+ uint8_t* data;
+ ASSERT_OK(pool->Allocate(100, &data));
+ EXPECT_EQ(static_cast<uint64_t>(0), reinterpret_cast<uint64_t>(data) % 64);
+ ASSERT_EQ(100, pool->bytes_allocated());
+
+ pool->Free(data, 100);
+ ASSERT_EQ(0, pool->bytes_allocated());
+}
+
+TEST(DefaultMemoryPool, OOM) {
+ MemoryPool* pool = default_memory_pool();
+
+ uint8_t* data;
+ int64_t to_alloc = std::numeric_limits<int64_t>::max();
+ ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
+}
+
+// Death tests and valgrind are known to not play well 100% of the time. See
+// googletest documentation
+#ifndef ARROW_VALGRIND
+
+TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
+ MemoryPool* pool = default_memory_pool();
+
+ uint8_t* data;
+ ASSERT_OK(pool->Allocate(100, &data));
+
+#ifndef NDEBUG
+ EXPECT_EXIT(pool->Free(data, 120), ::testing::ExitedWithCode(1),
+ ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
+#endif
+
+ pool->Free(data, 100);
+}
+
+#endif // ARROW_VALGRIND
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/memory_pool.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
new file mode 100644
index 0000000..f55b1ac
--- /dev/null
+++ b/cpp/src/arrow/memory_pool.cc
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/memory_pool.h"
+
+#include <cstdlib>
+#include <mutex>
+#include <sstream>
+#include <stdlib.h>
+
+#include "arrow/status.h"
+#include "arrow/util/logging.h"
+
+namespace arrow {
+
+namespace {
+// Allocate memory according to the alignment requirements for Arrow
+// (as of May 2016 64 bytes)
+Status AllocateAligned(int64_t size, uint8_t** out) {
+ // TODO(emkornfield) find something compatible with windows
+ constexpr size_t kAlignment = 64;
+#ifdef _MSC_VER
+ // Special code path for MSVC
+ *out = reinterpret_cast<uint8_t*>(_aligned_malloc(size, kAlignment));
+ if (!*out) {
+ std::stringstream ss;
+ ss << "malloc of size " << size << " failed";
+ return Status::OutOfMemory(ss.str());
+ }
+#else
+ const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment, size);
+ if (result == ENOMEM) {
+ std::stringstream ss;
+ ss << "malloc of size " << size << " failed";
+ return Status::OutOfMemory(ss.str());
+ }
+
+ if (result == EINVAL) {
+ std::stringstream ss;
+ ss << "invalid alignment parameter: " << kAlignment;
+ return Status::Invalid(ss.str());
+ }
+#endif
+ return Status::OK();
+}
+} // namespace
+
+MemoryPool::~MemoryPool() {}
+
+class InternalMemoryPool : public MemoryPool {
+ public:
+ InternalMemoryPool() : bytes_allocated_(0) {}
+ virtual ~InternalMemoryPool();
+
+ Status Allocate(int64_t size, uint8_t** out) override;
+
+ void Free(uint8_t* buffer, int64_t size) override;
+
+ int64_t bytes_allocated() const override;
+
+ private:
+ mutable std::mutex pool_lock_;
+ int64_t bytes_allocated_;
+};
+
+Status InternalMemoryPool::Allocate(int64_t size, uint8_t** out) {
+ std::lock_guard<std::mutex> guard(pool_lock_);
+ RETURN_NOT_OK(AllocateAligned(size, out));
+ bytes_allocated_ += size;
+
+ return Status::OK();
+}
+
+int64_t InternalMemoryPool::bytes_allocated() const {
+ std::lock_guard<std::mutex> guard(pool_lock_);
+ return bytes_allocated_;
+}
+
+void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
+ std::lock_guard<std::mutex> guard(pool_lock_);
+ DCHECK_GE(bytes_allocated_, size);
+#ifdef _MSC_VER
+ _aligned_free(buffer);
+#else
+ std::free(buffer);
+#endif
+ bytes_allocated_ -= size;
+}
+
+InternalMemoryPool::~InternalMemoryPool() {}
+
+MemoryPool* default_memory_pool() {
+ static InternalMemoryPool default_memory_pool_;
+ return &default_memory_pool_;
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/memory_pool.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
new file mode 100644
index 0000000..4c1d699
--- /dev/null
+++ b/cpp/src/arrow/memory_pool.h
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef ARROW_UTIL_MEMORY_POOL_H
+#define ARROW_UTIL_MEMORY_POOL_H
+
+#include <cstdint>
+
+#include "arrow/util/visibility.h"
+
+namespace arrow {
+
+class Status;
+
+class ARROW_EXPORT MemoryPool {
+ public:
+ virtual ~MemoryPool();
+
+ virtual Status Allocate(int64_t size, uint8_t** out) = 0;
+ virtual void Free(uint8_t* buffer, int64_t size) = 0;
+
+ virtual int64_t bytes_allocated() const = 0;
+};
+
+ARROW_EXPORT MemoryPool* default_memory_pool();
+
+} // namespace arrow
+
+#endif // ARROW_UTIL_MEMORY_POOL_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/pretty_print-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/pretty_print-test.cc b/cpp/src/arrow/pretty_print-test.cc
index b1e6a11..c22d3aa 100644
--- a/cpp/src/arrow/pretty_print-test.cc
+++ b/cpp/src/arrow/pretty_print-test.cc
@@ -26,14 +26,11 @@
#include "gtest/gtest.h"
#include "arrow/array.h"
+#include "arrow/builder.h"
#include "arrow/pretty_print.h"
#include "arrow/test-util.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/pretty_print.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index c63a9e9..9c439c4 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -22,13 +22,10 @@
#include "arrow/array.h"
#include "arrow/pretty_print.h"
+#include "arrow/status.h"
#include "arrow/table.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
-#include "arrow/types/list.h"
-#include "arrow/types/string.h"
-#include "arrow/types/struct.h"
-#include "arrow/util/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/status-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/status-test.cc b/cpp/src/arrow/status-test.cc
new file mode 100644
index 0000000..969ba97
--- /dev/null
+++ b/cpp/src/arrow/status-test.cc
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "gtest/gtest.h"
+
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+
+namespace arrow {
+
+TEST(StatusTest, TestCodeAndMessage) {
+ Status ok = Status::OK();
+ ASSERT_EQ(StatusCode::OK, ok.code());
+ Status file_error = Status::IOError("file error");
+ ASSERT_EQ(StatusCode::IOError, file_error.code());
+ ASSERT_EQ("file error", file_error.message());
+}
+
+TEST(StatusTest, TestToString) {
+ Status file_error = Status::IOError("file error");
+ ASSERT_EQ("IOError: file error", file_error.ToString());
+}
+
+} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/status.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/status.cc b/cpp/src/arrow/status.cc
new file mode 100644
index 0000000..e1a2427
--- /dev/null
+++ b/cpp/src/arrow/status.cc
@@ -0,0 +1,86 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// A Status encapsulates the result of an operation. It may indicate success,
+// or it may indicate an error with an associated error message.
+//
+// Multiple threads can invoke const methods on a Status without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same Status must use
+// external synchronization.
+
+#include "arrow/status.h"
+
+#include <assert.h>
+
+namespace arrow {
+
+Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
+ assert(code != StatusCode::OK);
+ const uint32_t size = msg.size();
+ char* result = new char[size + 7];
+ memcpy(result, &size, sizeof(size));
+ result[4] = static_cast<char>(code);
+ memcpy(result + 5, &posix_code, sizeof(posix_code));
+ memcpy(result + 7, msg.c_str(), msg.size());
+ state_ = result;
+}
+
+const char* Status::CopyState(const char* state) {
+ uint32_t size;
+ memcpy(&size, state, sizeof(size));
+ char* result = new char[size + 7];
+ memcpy(result, state, size + 7);
+ return result;
+}
+
+std::string Status::CodeAsString() const {
+ if (state_ == NULL) { return "OK"; }
+
+ const char* type;
+ switch (code()) {
+ case StatusCode::OK:
+ type = "OK";
+ break;
+ case StatusCode::OutOfMemory:
+ type = "Out of memory";
+ break;
+ case StatusCode::KeyError:
+ type = "Key error";
+ break;
+ case StatusCode::TypeError:
+ type = "Type error";
+ break;
+ case StatusCode::Invalid:
+ type = "Invalid";
+ break;
+ case StatusCode::IOError:
+ type = "IOError";
+ break;
+ case StatusCode::UnknownError:
+ type = "Unknown error";
+ break;
+ case StatusCode::NotImplemented:
+ type = "NotImplemented";
+ break;
+ default:
+ type = "Unknown";
+ break;
+ }
+ return std::string(type);
+}
+
+std::string Status::ToString() const {
+ std::string result(CodeAsString());
+ if (state_ == NULL) { return result; }
+
+ result.append(": ");
+
+ uint32_t length;
+ memcpy(&length, state_, sizeof(length));
+ result.append(reinterpret_cast<const char*>(state_ + 7), length);
+ return result;
+}
+
+} // namespace arrow
[2/5] arrow git commit: ARROW-418: [C++] Array / Builder class code
reorganization, flattening
Posted by we...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/string-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc
deleted file mode 100644
index 3c4b12b..0000000
--- a/cpp/src/arrow/types/string-test.cc
+++ /dev/null
@@ -1,360 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdlib>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/string.h"
-#include "arrow/types/test-common.h"
-
-namespace arrow {
-
-class Buffer;
-
-TEST(TypesTest, BinaryType) {
- BinaryType t1;
- BinaryType e1;
- StringType t2;
- EXPECT_TRUE(t1.Equals(&e1));
- EXPECT_FALSE(t1.Equals(&t2));
- ASSERT_EQ(t1.type, Type::BINARY);
- ASSERT_EQ(t1.ToString(), std::string("binary"));
-}
-
-TEST(TypesTest, TestStringType) {
- StringType str;
- ASSERT_EQ(str.type, Type::STRING);
- ASSERT_EQ(str.ToString(), std::string("string"));
-}
-
-// ----------------------------------------------------------------------
-// String container
-
-class TestStringContainer : public ::testing::Test {
- public:
- void SetUp() {
- chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
- offsets_ = {0, 1, 1, 1, 3, 6};
- valid_bytes_ = {1, 1, 0, 1, 1};
- expected_ = {"a", "", "", "bb", "ccc"};
-
- MakeArray();
- }
-
- void MakeArray() {
- length_ = offsets_.size() - 1;
- value_buf_ = test::GetBufferFromVector(chars_);
- offsets_buf_ = test::GetBufferFromVector(offsets_);
- null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
- null_count_ = test::null_count(valid_bytes_);
-
- strings_ = std::make_shared<StringArray>(
- length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
- }
-
- protected:
- std::vector<int32_t> offsets_;
- std::vector<char> chars_;
- std::vector<uint8_t> valid_bytes_;
-
- std::vector<std::string> expected_;
-
- std::shared_ptr<Buffer> value_buf_;
- std::shared_ptr<Buffer> offsets_buf_;
- std::shared_ptr<Buffer> null_bitmap_;
-
- int null_count_;
- int length_;
-
- std::shared_ptr<StringArray> strings_;
-};
-
-TEST_F(TestStringContainer, TestArrayBasics) {
- ASSERT_EQ(length_, strings_->length());
- ASSERT_EQ(1, strings_->null_count());
- ASSERT_OK(strings_->Validate());
-}
-
-TEST_F(TestStringContainer, TestType) {
- TypePtr type = strings_->type();
-
- ASSERT_EQ(Type::STRING, type->type);
- ASSERT_EQ(Type::STRING, strings_->type_enum());
-}
-
-TEST_F(TestStringContainer, TestListFunctions) {
- int pos = 0;
- for (size_t i = 0; i < expected_.size(); ++i) {
- ASSERT_EQ(pos, strings_->value_offset(i));
- ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
- pos += expected_[i].size();
- }
-}
-
-TEST_F(TestStringContainer, TestDestructor) {
- auto arr = std::make_shared<StringArray>(
- length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
-}
-
-TEST_F(TestStringContainer, TestGetString) {
- for (size_t i = 0; i < expected_.size(); ++i) {
- if (valid_bytes_[i] == 0) {
- ASSERT_TRUE(strings_->IsNull(i));
- } else {
- ASSERT_EQ(expected_[i], strings_->GetString(i));
- }
- }
-}
-
-TEST_F(TestStringContainer, TestEmptyStringComparison) {
- offsets_ = {0, 0, 0, 0, 0, 0};
- offsets_buf_ = test::GetBufferFromVector(offsets_);
- length_ = offsets_.size() - 1;
-
- auto strings_a = std::make_shared<StringArray>(
- length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
- auto strings_b = std::make_shared<StringArray>(
- length_, offsets_buf_, nullptr, null_count_, null_bitmap_);
- ASSERT_TRUE(strings_a->Equals(strings_b));
-}
-
-// ----------------------------------------------------------------------
-// String builder tests
-
-class TestStringBuilder : public TestBuilder {
- public:
- void SetUp() {
- TestBuilder::SetUp();
- type_ = TypePtr(new StringType());
- builder_.reset(new StringBuilder(pool_, type_));
- }
-
- void Done() {
- std::shared_ptr<Array> out;
- EXPECT_OK(builder_->Finish(&out));
-
- result_ = std::dynamic_pointer_cast<StringArray>(out);
- result_->Validate();
- }
-
- protected:
- TypePtr type_;
-
- std::unique_ptr<StringBuilder> builder_;
- std::shared_ptr<StringArray> result_;
-};
-
-TEST_F(TestStringBuilder, TestScalarAppend) {
- std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
- std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
-
- int N = strings.size();
- int reps = 1000;
-
- for (int j = 0; j < reps; ++j) {
- for (int i = 0; i < N; ++i) {
- if (is_null[i]) {
- builder_->AppendNull();
- } else {
- builder_->Append(strings[i]);
- }
- }
- }
- Done();
-
- ASSERT_EQ(reps * N, result_->length());
- ASSERT_EQ(reps, result_->null_count());
- ASSERT_EQ(reps * 6, result_->data()->size());
-
- int32_t length;
- int32_t pos = 0;
- for (int i = 0; i < N * reps; ++i) {
- if (is_null[i % N]) {
- ASSERT_TRUE(result_->IsNull(i));
- } else {
- ASSERT_FALSE(result_->IsNull(i));
- result_->GetValue(i, &length);
- ASSERT_EQ(pos, result_->offset(i));
- ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
- ASSERT_EQ(strings[i % N], result_->GetString(i));
-
- pos += length;
- }
- }
-}
-
-TEST_F(TestStringBuilder, TestZeroLength) {
- // All buffers are null
- Done();
-}
-
-// Binary container type
-// TODO(emkornfield) there should be some way to refactor these to avoid code duplicating
-// with String
-class TestBinaryContainer : public ::testing::Test {
- public:
- void SetUp() {
- chars_ = {'a', 'b', 'b', 'c', 'c', 'c'};
- offsets_ = {0, 1, 1, 1, 3, 6};
- valid_bytes_ = {1, 1, 0, 1, 1};
- expected_ = {"a", "", "", "bb", "ccc"};
-
- MakeArray();
- }
-
- void MakeArray() {
- length_ = offsets_.size() - 1;
- value_buf_ = test::GetBufferFromVector(chars_);
- offsets_buf_ = test::GetBufferFromVector(offsets_);
-
- null_bitmap_ = test::bytes_to_null_buffer(valid_bytes_);
- null_count_ = test::null_count(valid_bytes_);
-
- strings_ = std::make_shared<BinaryArray>(
- length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
- }
-
- protected:
- std::vector<int32_t> offsets_;
- std::vector<char> chars_;
- std::vector<uint8_t> valid_bytes_;
-
- std::vector<std::string> expected_;
-
- std::shared_ptr<Buffer> value_buf_;
- std::shared_ptr<Buffer> offsets_buf_;
- std::shared_ptr<Buffer> null_bitmap_;
-
- int null_count_;
- int length_;
-
- std::shared_ptr<BinaryArray> strings_;
-};
-
-TEST_F(TestBinaryContainer, TestArrayBasics) {
- ASSERT_EQ(length_, strings_->length());
- ASSERT_EQ(1, strings_->null_count());
- ASSERT_OK(strings_->Validate());
-}
-
-TEST_F(TestBinaryContainer, TestType) {
- TypePtr type = strings_->type();
-
- ASSERT_EQ(Type::BINARY, type->type);
- ASSERT_EQ(Type::BINARY, strings_->type_enum());
-}
-
-TEST_F(TestBinaryContainer, TestListFunctions) {
- int pos = 0;
- for (size_t i = 0; i < expected_.size(); ++i) {
- ASSERT_EQ(pos, strings_->value_offset(i));
- ASSERT_EQ(static_cast<int>(expected_[i].size()), strings_->value_length(i));
- pos += expected_[i].size();
- }
-}
-
-TEST_F(TestBinaryContainer, TestDestructor) {
- auto arr = std::make_shared<BinaryArray>(
- length_, offsets_buf_, value_buf_, null_count_, null_bitmap_);
-}
-
-TEST_F(TestBinaryContainer, TestGetValue) {
- for (size_t i = 0; i < expected_.size(); ++i) {
- if (valid_bytes_[i] == 0) {
- ASSERT_TRUE(strings_->IsNull(i));
- } else {
- int32_t len = -1;
- const uint8_t* bytes = strings_->GetValue(i, &len);
- ASSERT_EQ(0, std::memcmp(expected_[i].data(), bytes, len));
- }
- }
-}
-
-class TestBinaryBuilder : public TestBuilder {
- public:
- void SetUp() {
- TestBuilder::SetUp();
- type_ = TypePtr(new BinaryType());
- builder_.reset(new BinaryBuilder(pool_, type_));
- }
-
- void Done() {
- std::shared_ptr<Array> out;
- EXPECT_OK(builder_->Finish(&out));
-
- result_ = std::dynamic_pointer_cast<BinaryArray>(out);
- result_->Validate();
- }
-
- protected:
- TypePtr type_;
-
- std::unique_ptr<BinaryBuilder> builder_;
- std::shared_ptr<BinaryArray> result_;
-};
-
-TEST_F(TestBinaryBuilder, TestScalarAppend) {
- std::vector<std::string> strings = {"", "bb", "a", "", "ccc"};
- std::vector<uint8_t> is_null = {0, 0, 0, 1, 0};
-
- int N = strings.size();
- int reps = 1000;
-
- for (int j = 0; j < reps; ++j) {
- for (int i = 0; i < N; ++i) {
- if (is_null[i]) {
- builder_->AppendNull();
- } else {
- builder_->Append(
- reinterpret_cast<const uint8_t*>(strings[i].data()), strings[i].size());
- }
- }
- }
- Done();
- ASSERT_OK(result_->Validate());
- ASSERT_EQ(reps * N, result_->length());
- ASSERT_EQ(reps, result_->null_count());
- ASSERT_EQ(reps * 6, result_->data()->size());
-
- int32_t length;
- for (int i = 0; i < N * reps; ++i) {
- if (is_null[i % N]) {
- ASSERT_TRUE(result_->IsNull(i));
- } else {
- ASSERT_FALSE(result_->IsNull(i));
- const uint8_t* vals = result_->GetValue(i, &length);
- ASSERT_EQ(static_cast<int>(strings[i % N].size()), length);
- ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length));
- }
- }
-}
-
-TEST_F(TestBinaryBuilder, TestZeroLength) {
- // All buffers are null
- Done();
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/string.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc
deleted file mode 100644
index db963df..0000000
--- a/cpp/src/arrow/types/string.cc
+++ /dev/null
@@ -1,150 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/string.h"
-
-#include <cstring>
-#include <sstream>
-#include <string>
-
-#include "arrow/type.h"
-
-namespace arrow {
-
-static std::shared_ptr<DataType> kBinary = std::make_shared<BinaryType>();
-static std::shared_ptr<DataType> kString = std::make_shared<StringType>();
-
-BinaryArray::BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
- const std::shared_ptr<Buffer>& data, int32_t null_count,
- const std::shared_ptr<Buffer>& null_bitmap)
- : BinaryArray(kBinary, length, offsets, data, null_count, null_bitmap) {}
-
-BinaryArray::BinaryArray(const TypePtr& type, int32_t length,
- const std::shared_ptr<Buffer>& offsets, const std::shared_ptr<Buffer>& data,
- int32_t null_count, const std::shared_ptr<Buffer>& null_bitmap)
- : Array(type, length, null_count, null_bitmap),
- offset_buffer_(offsets),
- offsets_(reinterpret_cast<const int32_t*>(offset_buffer_->data())),
- data_buffer_(data),
- data_(nullptr) {
- if (data_buffer_ != nullptr) { data_ = data_buffer_->data(); }
-}
-
-Status BinaryArray::Validate() const {
- // TODO(wesm): what to do here?
- return Status::OK();
-}
-
-bool BinaryArray::EqualsExact(const BinaryArray& other) const {
- if (!Array::EqualsExact(other)) { return false; }
-
- bool equal_offsets =
- offset_buffer_->Equals(*other.offset_buffer_, (length_ + 1) * sizeof(int32_t));
- if (!equal_offsets) { return false; }
-
- if (!data_buffer_ && !(other.data_buffer_)) { return true; }
-
- return data_buffer_->Equals(*other.data_buffer_, data_buffer_->size());
-}
-
-bool BinaryArray::Equals(const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- return EqualsExact(*static_cast<const BinaryArray*>(arr.get()));
-}
-
-bool BinaryArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- const auto other = static_cast<const BinaryArray*>(arr.get());
- for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
- const bool is_null = IsNull(i);
- if (is_null != arr->IsNull(o_i)) { return false; }
- if (is_null) continue;
- const int32_t begin_offset = offset(i);
- const int32_t end_offset = offset(i + 1);
- const int32_t other_begin_offset = other->offset(o_i);
- const int32_t other_end_offset = other->offset(o_i + 1);
- // Underlying can't be equal if the size isn't equal
- if (end_offset - begin_offset != other_end_offset - other_begin_offset) {
- return false;
- }
-
- if (std::memcmp(data_ + begin_offset, other->data_ + other_begin_offset,
- end_offset - begin_offset)) {
- return false;
- }
- }
- return true;
-}
-
-Status BinaryArray::Accept(ArrayVisitor* visitor) const {
- return visitor->Visit(*this);
-}
-
-StringArray::StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
- const std::shared_ptr<Buffer>& data, int32_t null_count,
- const std::shared_ptr<Buffer>& null_bitmap)
- : BinaryArray(kString, length, offsets, data, null_count, null_bitmap) {}
-
-Status StringArray::Validate() const {
- // TODO(emkornfield) Validate proper UTF8 code points?
- return BinaryArray::Validate();
-}
-
-Status StringArray::Accept(ArrayVisitor* visitor) const {
- return visitor->Visit(*this);
-}
-
-// This used to be a static member variable of BinaryBuilder, but it can cause
-// valgrind to report a (spurious?) memory leak when needed in other shared
-// libraries. The problem came up while adding explicit visibility to libarrow
-// and libparquet_arrow
-static TypePtr kBinaryValueType = TypePtr(new UInt8Type());
-
-BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type)
- : ListBuilder(pool, std::make_shared<UInt8Builder>(pool, kBinaryValueType), type) {
- byte_builder_ = static_cast<UInt8Builder*>(value_builder_.get());
-}
-
-Status BinaryBuilder::Finish(std::shared_ptr<Array>* out) {
- std::shared_ptr<Array> result;
- RETURN_NOT_OK(ListBuilder::Finish(&result));
-
- const auto list = std::dynamic_pointer_cast<ListArray>(result);
- auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
-
- *out = std::make_shared<BinaryArray>(list->length(), list->offsets(), values->data(),
- list->null_count(), list->null_bitmap());
- return Status::OK();
-}
-
-Status StringBuilder::Finish(std::shared_ptr<Array>* out) {
- std::shared_ptr<Array> result;
- RETURN_NOT_OK(ListBuilder::Finish(&result));
-
- const auto list = std::dynamic_pointer_cast<ListArray>(result);
- auto values = std::dynamic_pointer_cast<UInt8Array>(list->values());
-
- *out = std::make_shared<StringArray>(list->length(), list->offsets(), values->data(),
- list->null_count(), list->null_bitmap());
- return Status::OK();
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/string.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h
deleted file mode 100644
index c875243..0000000
--- a/cpp/src/arrow/types/string.h
+++ /dev/null
@@ -1,149 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_STRING_H
-#define ARROW_TYPES_STRING_H
-
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Buffer;
-class MemoryPool;
-
-class ARROW_EXPORT BinaryArray : public Array {
- public:
- using TypeClass = BinaryType;
-
- BinaryArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
- const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
- const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
- // Constructor that allows sub-classes/builders to propagate there logical type up the
- // class hierarchy.
- BinaryArray(const TypePtr& type, int32_t length, const std::shared_ptr<Buffer>& offsets,
- const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
- const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
- // Return the pointer to the given elements bytes
- // TODO(emkornfield) introduce a StringPiece or something similar to capture zero-copy
- // pointer + offset
- const uint8_t* GetValue(int i, int32_t* out_length) const {
- DCHECK(out_length);
- const int32_t pos = offsets_[i];
- *out_length = offsets_[i + 1] - pos;
- return data_ + pos;
- }
-
- std::shared_ptr<Buffer> data() const { return data_buffer_; }
- std::shared_ptr<Buffer> offsets() const { return offset_buffer_; }
-
- const int32_t* raw_offsets() const { return offsets_; }
-
- int32_t offset(int i) const { return offsets_[i]; }
-
- // Neither of these functions will perform boundschecking
- int32_t value_offset(int i) const { return offsets_[i]; }
- int32_t value_length(int i) const { return offsets_[i + 1] - offsets_[i]; }
-
- bool EqualsExact(const BinaryArray& other) const;
- bool Equals(const std::shared_ptr<Array>& arr) const override;
- bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const ArrayPtr& arr) const override;
-
- Status Validate() const override;
-
- Status Accept(ArrayVisitor* visitor) const override;
-
- private:
- std::shared_ptr<Buffer> offset_buffer_;
- const int32_t* offsets_;
-
- std::shared_ptr<Buffer> data_buffer_;
- const uint8_t* data_;
-};
-
-class ARROW_EXPORT StringArray : public BinaryArray {
- public:
- using TypeClass = StringType;
-
- StringArray(int32_t length, const std::shared_ptr<Buffer>& offsets,
- const std::shared_ptr<Buffer>& data, int32_t null_count = 0,
- const std::shared_ptr<Buffer>& null_bitmap = nullptr);
-
- // Construct a std::string
- // TODO: std::bad_alloc possibility
- std::string GetString(int i) const {
- int32_t nchars;
- const uint8_t* str = GetValue(i, &nchars);
- return std::string(reinterpret_cast<const char*>(str), nchars);
- }
-
- Status Validate() const override;
-
- Status Accept(ArrayVisitor* visitor) const override;
-};
-
-// BinaryBuilder : public ListBuilder
-class ARROW_EXPORT BinaryBuilder : public ListBuilder {
- public:
- explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type);
- virtual ~BinaryBuilder() {}
-
- Status Append(const uint8_t* value, int32_t length) {
- RETURN_NOT_OK(ListBuilder::Append());
- return byte_builder_->Append(value, length);
- }
-
- Status Append(const char* value, int32_t length) {
- return Append(reinterpret_cast<const uint8_t*>(value), length);
- }
-
- Status Append(const std::string& value) { return Append(value.c_str(), value.size()); }
-
- Status Finish(std::shared_ptr<Array>* out) override;
-
- protected:
- UInt8Builder* byte_builder_;
-};
-
-// String builder
-class ARROW_EXPORT StringBuilder : public BinaryBuilder {
- public:
- explicit StringBuilder(MemoryPool* pool, const TypePtr& type)
- : BinaryBuilder(pool, type) {}
-
- using BinaryBuilder::Append;
-
- Status Finish(std::shared_ptr<Array>* out) override;
-
- Status Append(const std::vector<std::string>& values, uint8_t* null_bytes);
-};
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_STRING_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/struct-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc
deleted file mode 100644
index 197d7d4..0000000
--- a/cpp/src/arrow/types/struct-test.cc
+++ /dev/null
@@ -1,396 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/types/construct.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/types/struct.h"
-#include "arrow/types/test-common.h"
-#include "arrow/util/status.h"
-
-using std::shared_ptr;
-using std::string;
-using std::vector;
-
-namespace arrow {
-
-TEST(TestStructType, Basics) {
- TypePtr f0_type = TypePtr(new Int32Type());
- auto f0 = std::make_shared<Field>("f0", f0_type);
-
- TypePtr f1_type = TypePtr(new StringType());
- auto f1 = std::make_shared<Field>("f1", f1_type);
-
- TypePtr f2_type = TypePtr(new UInt8Type());
- auto f2 = std::make_shared<Field>("f2", f2_type);
-
- vector<shared_ptr<Field>> fields = {f0, f1, f2};
-
- StructType struct_type(fields);
-
- ASSERT_TRUE(struct_type.child(0)->Equals(f0));
- ASSERT_TRUE(struct_type.child(1)->Equals(f1));
- ASSERT_TRUE(struct_type.child(2)->Equals(f2));
-
- ASSERT_EQ(struct_type.ToString(), "struct<f0: int32, f1: string, f2: uint8>");
-
- // TODO(wesm): out of bounds for field(...)
-}
-
-void ValidateBasicStructArray(const StructArray* result,
- const vector<uint8_t>& struct_is_valid, const vector<char>& list_values,
- const vector<uint8_t>& list_is_valid, const vector<int>& list_lengths,
- const vector<int>& list_offsets, const vector<int32_t>& int_values) {
- ASSERT_EQ(4, result->length());
- ASSERT_OK(result->Validate());
-
- auto list_char_arr = static_cast<ListArray*>(result->field(0).get());
- auto char_arr = static_cast<Int8Array*>(list_char_arr->values().get());
- auto int32_arr = static_cast<Int32Array*>(result->field(1).get());
-
- ASSERT_EQ(0, result->null_count());
- ASSERT_EQ(1, list_char_arr->null_count());
- ASSERT_EQ(0, int32_arr->null_count());
-
- // List<char>
- ASSERT_EQ(4, list_char_arr->length());
- ASSERT_EQ(10, list_char_arr->values()->length());
- for (size_t i = 0; i < list_offsets.size(); ++i) {
- ASSERT_EQ(list_offsets[i], list_char_arr->raw_offsets()[i]);
- }
- for (size_t i = 0; i < list_values.size(); ++i) {
- ASSERT_EQ(list_values[i], char_arr->Value(i));
- }
-
- // Int32
- ASSERT_EQ(4, int32_arr->length());
- for (size_t i = 0; i < int_values.size(); ++i) {
- ASSERT_EQ(int_values[i], int32_arr->Value(i));
- }
-}
-
-// ----------------------------------------------------------------------------------
-// Struct test
-class TestStructBuilder : public TestBuilder {
- public:
- void SetUp() {
- TestBuilder::SetUp();
-
- auto int32_type = TypePtr(new Int32Type());
- auto char_type = TypePtr(new Int8Type());
- auto list_type = TypePtr(new ListType(char_type));
-
- std::vector<TypePtr> types = {list_type, int32_type};
- std::vector<FieldPtr> fields;
- fields.push_back(FieldPtr(new Field("list", list_type)));
- fields.push_back(FieldPtr(new Field("int", int32_type)));
-
- type_ = TypePtr(new StructType(fields));
- value_fields_ = fields;
-
- std::shared_ptr<ArrayBuilder> tmp;
- ASSERT_OK(MakeBuilder(pool_, type_, &tmp));
-
- builder_ = std::dynamic_pointer_cast<StructBuilder>(tmp);
- ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
- }
-
- void Done() {
- std::shared_ptr<Array> out;
- ASSERT_OK(builder_->Finish(&out));
- result_ = std::dynamic_pointer_cast<StructArray>(out);
- }
-
- protected:
- std::vector<FieldPtr> value_fields_;
- TypePtr type_;
-
- std::shared_ptr<StructBuilder> builder_;
- std::shared_ptr<StructArray> result_;
-};
-
-TEST_F(TestStructBuilder, TestAppendNull) {
- ASSERT_OK(builder_->AppendNull());
- ASSERT_OK(builder_->AppendNull());
- ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
-
- ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
- ASSERT_OK(list_vb->AppendNull());
- ASSERT_OK(list_vb->AppendNull());
- ASSERT_EQ(2, list_vb->length());
-
- Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
- ASSERT_OK(int_vb->AppendNull());
- ASSERT_OK(int_vb->AppendNull());
- ASSERT_EQ(2, int_vb->length());
-
- Done();
-
- ASSERT_OK(result_->Validate());
-
- ASSERT_EQ(2, static_cast<int>(result_->fields().size()));
- ASSERT_EQ(2, result_->length());
- ASSERT_EQ(2, result_->field(0)->length());
- ASSERT_EQ(2, result_->field(1)->length());
- ASSERT_TRUE(result_->IsNull(0));
- ASSERT_TRUE(result_->IsNull(1));
- ASSERT_TRUE(result_->field(0)->IsNull(0));
- ASSERT_TRUE(result_->field(0)->IsNull(1));
- ASSERT_TRUE(result_->field(1)->IsNull(0));
- ASSERT_TRUE(result_->field(1)->IsNull(1));
-
- ASSERT_EQ(Type::LIST, result_->field(0)->type_enum());
- ASSERT_EQ(Type::INT32, result_->field(1)->type_enum());
-}
-
-TEST_F(TestStructBuilder, TestBasics) {
- vector<int32_t> int_values = {1, 2, 3, 4};
- vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
- vector<int> list_lengths = {3, 0, 3, 4};
- vector<int> list_offsets = {0, 3, 3, 6, 10};
- vector<uint8_t> list_is_valid = {1, 0, 1, 1};
- vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
-
- ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
- Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
- Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
- ASSERT_EQ(2, static_cast<int>(builder_->field_builders().size()));
-
- EXPECT_OK(builder_->Resize(list_lengths.size()));
- EXPECT_OK(char_vb->Resize(list_values.size()));
- EXPECT_OK(int_vb->Resize(int_values.size()));
-
- int pos = 0;
- for (size_t i = 0; i < list_lengths.size(); ++i) {
- ASSERT_OK(list_vb->Append(list_is_valid[i] > 0));
- int_vb->UnsafeAppend(int_values[i]);
- for (int j = 0; j < list_lengths[i]; ++j) {
- char_vb->UnsafeAppend(list_values[pos++]);
- }
- }
-
- for (size_t i = 0; i < struct_is_valid.size(); ++i) {
- ASSERT_OK(builder_->Append(struct_is_valid[i] > 0));
- }
-
- Done();
-
- ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
- list_lengths, list_offsets, int_values);
-}
-
-TEST_F(TestStructBuilder, BulkAppend) {
- vector<int32_t> int_values = {1, 2, 3, 4};
- vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
- vector<int> list_lengths = {3, 0, 3, 4};
- vector<int> list_offsets = {0, 3, 3, 6};
- vector<uint8_t> list_is_valid = {1, 0, 1, 1};
- vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
-
- ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
- Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
- Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-
- ASSERT_OK(builder_->Resize(list_lengths.size()));
- ASSERT_OK(char_vb->Resize(list_values.size()));
- ASSERT_OK(int_vb->Resize(int_values.size()));
-
- builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-
- list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
- for (int8_t value : list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- Done();
- ValidateBasicStructArray(result_.get(), struct_is_valid, list_values, list_is_valid,
- list_lengths, list_offsets, int_values);
-}
-
-TEST_F(TestStructBuilder, BulkAppendInvalid) {
- vector<int32_t> int_values = {1, 2, 3, 4};
- vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
- vector<int> list_lengths = {3, 0, 3, 4};
- vector<int> list_offsets = {0, 3, 3, 6};
- vector<uint8_t> list_is_valid = {1, 0, 1, 1};
- vector<uint8_t> struct_is_valid = {1, 0, 1, 1}; // should be 1, 1, 1, 1
-
- ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
- Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
- Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
-
- ASSERT_OK(builder_->Reserve(list_lengths.size()));
- ASSERT_OK(char_vb->Reserve(list_values.size()));
- ASSERT_OK(int_vb->Reserve(int_values.size()));
-
- builder_->Append(struct_is_valid.size(), struct_is_valid.data());
-
- list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
- for (int8_t value : list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- Done();
- // Even null bitmap of the parent Struct is not valid, Validate() will ignore it.
- ASSERT_OK(result_->Validate());
-}
-
-TEST_F(TestStructBuilder, TestEquality) {
- ArrayPtr array, equal_array;
- ArrayPtr unequal_bitmap_array, unequal_offsets_array, unequal_values_array;
-
- vector<int32_t> int_values = {1, 2, 3, 4};
- vector<char> list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'm', 'a', 'r', 'k'};
- vector<int> list_lengths = {3, 0, 3, 4};
- vector<int> list_offsets = {0, 3, 3, 6};
- vector<uint8_t> list_is_valid = {1, 0, 1, 1};
- vector<uint8_t> struct_is_valid = {1, 1, 1, 1};
-
- vector<int32_t> unequal_int_values = {4, 2, 3, 1};
- vector<char> unequal_list_values = {'j', 'o', 'e', 'b', 'o', 'b', 'l', 'u', 'c', 'y'};
- vector<int> unequal_list_offsets = {0, 3, 4, 6};
- vector<uint8_t> unequal_list_is_valid = {1, 1, 1, 1};
- vector<uint8_t> unequal_struct_is_valid = {1, 0, 0, 1};
-
- ListBuilder* list_vb = static_cast<ListBuilder*>(builder_->field_builder(0).get());
- Int8Builder* char_vb = static_cast<Int8Builder*>(list_vb->value_builder().get());
- Int32Builder* int_vb = static_cast<Int32Builder*>(builder_->field_builder(1).get());
- ASSERT_OK(builder_->Reserve(list_lengths.size()));
- ASSERT_OK(char_vb->Reserve(list_values.size()));
- ASSERT_OK(int_vb->Reserve(int_values.size()));
-
- // setup two equal arrays, one of which takes an unequal bitmap
- builder_->Append(struct_is_valid.size(), struct_is_valid.data());
- list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
- for (int8_t value : list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- ASSERT_OK(builder_->Finish(&array));
-
- ASSERT_OK(builder_->Resize(list_lengths.size()));
- ASSERT_OK(char_vb->Resize(list_values.size()));
- ASSERT_OK(int_vb->Resize(int_values.size()));
-
- builder_->Append(struct_is_valid.size(), struct_is_valid.data());
- list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
- for (int8_t value : list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- ASSERT_OK(builder_->Finish(&equal_array));
-
- ASSERT_OK(builder_->Resize(list_lengths.size()));
- ASSERT_OK(char_vb->Resize(list_values.size()));
- ASSERT_OK(int_vb->Resize(int_values.size()));
-
- // setup an unequal one with the unequal bitmap
- builder_->Append(unequal_struct_is_valid.size(), unequal_struct_is_valid.data());
- list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
- for (int8_t value : list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- ASSERT_OK(builder_->Finish(&unequal_bitmap_array));
-
- ASSERT_OK(builder_->Resize(list_lengths.size()));
- ASSERT_OK(char_vb->Resize(list_values.size()));
- ASSERT_OK(int_vb->Resize(int_values.size()));
-
- // setup an unequal one with unequal offsets
- builder_->Append(struct_is_valid.size(), struct_is_valid.data());
- list_vb->Append(unequal_list_offsets.data(), unequal_list_offsets.size(),
- unequal_list_is_valid.data());
- for (int8_t value : list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- ASSERT_OK(builder_->Finish(&unequal_offsets_array));
-
- ASSERT_OK(builder_->Resize(list_lengths.size()));
- ASSERT_OK(char_vb->Resize(list_values.size()));
- ASSERT_OK(int_vb->Resize(int_values.size()));
-
- // setup anunequal one with unequal values
- builder_->Append(struct_is_valid.size(), struct_is_valid.data());
- list_vb->Append(list_offsets.data(), list_offsets.size(), list_is_valid.data());
- for (int8_t value : unequal_list_values) {
- char_vb->UnsafeAppend(value);
- }
- for (int32_t value : unequal_int_values) {
- int_vb->UnsafeAppend(value);
- }
-
- ASSERT_OK(builder_->Finish(&unequal_values_array));
-
- // Test array equality
- EXPECT_TRUE(array->Equals(array));
- EXPECT_TRUE(array->Equals(equal_array));
- EXPECT_TRUE(equal_array->Equals(array));
- EXPECT_FALSE(equal_array->Equals(unequal_bitmap_array));
- EXPECT_FALSE(unequal_bitmap_array->Equals(equal_array));
- EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_values_array));
- EXPECT_FALSE(unequal_values_array->Equals(unequal_bitmap_array));
- EXPECT_FALSE(unequal_bitmap_array->Equals(unequal_offsets_array));
- EXPECT_FALSE(unequal_offsets_array->Equals(unequal_bitmap_array));
-
- // Test range equality
- EXPECT_TRUE(array->RangeEquals(0, 4, 0, equal_array));
- EXPECT_TRUE(array->RangeEquals(3, 4, 3, unequal_bitmap_array));
- EXPECT_TRUE(array->RangeEquals(0, 1, 0, unequal_offsets_array));
- EXPECT_FALSE(array->RangeEquals(0, 2, 0, unequal_offsets_array));
- EXPECT_FALSE(array->RangeEquals(1, 2, 1, unequal_offsets_array));
- EXPECT_FALSE(array->RangeEquals(0, 1, 0, unequal_values_array));
- EXPECT_TRUE(array->RangeEquals(1, 3, 1, unequal_values_array));
- EXPECT_FALSE(array->RangeEquals(3, 4, 3, unequal_values_array));
-}
-
-TEST_F(TestStructBuilder, TestZeroLength) {
- // All buffers are null
- Done();
- ASSERT_OK(result_->Validate());
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/struct.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/struct.cc b/cpp/src/arrow/types/struct.cc
deleted file mode 100644
index 0e0db23..0000000
--- a/cpp/src/arrow/types/struct.cc
+++ /dev/null
@@ -1,108 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/struct.h"
-
-#include <sstream>
-
-namespace arrow {
-
-bool StructArray::Equals(const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (this->type_enum() != arr->type_enum()) { return false; }
- if (null_count_ != arr->null_count()) { return false; }
- return RangeEquals(0, length_, 0, arr);
-}
-
-bool StructArray::RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const std::shared_ptr<Array>& arr) const {
- if (this == arr.get()) { return true; }
- if (!arr) { return false; }
- if (Type::STRUCT != arr->type_enum()) { return false; }
- const auto other = static_cast<StructArray*>(arr.get());
-
- bool equal_fields = true;
- for (int32_t i = start_idx, o_i = other_start_idx; i < end_idx; ++i, ++o_i) {
- if (IsNull(i) != arr->IsNull(o_i)) { return false; }
- if (IsNull(i)) continue;
- for (size_t j = 0; j < field_arrays_.size(); ++j) {
- // TODO: really we should be comparing stretches of non-null data rather
- // than looking at one value at a time.
- equal_fields = field(j)->RangeEquals(i, i + 1, o_i, other->field(j));
- if (!equal_fields) { return false; }
- }
- }
-
- return true;
-}
-
-Status StructArray::Validate() const {
- if (length_ < 0) { return Status::Invalid("Length was negative"); }
-
- if (null_count() > length_) {
- return Status::Invalid("Null count exceeds the length of this struct");
- }
-
- if (field_arrays_.size() > 0) {
- // Validate fields
- int32_t array_length = field_arrays_[0]->length();
- size_t idx = 0;
- for (auto it : field_arrays_) {
- if (it->length() != array_length) {
- std::stringstream ss;
- ss << "Length is not equal from field " << it->type()->ToString()
- << " at position {" << idx << "}";
- return Status::Invalid(ss.str());
- }
-
- const Status child_valid = it->Validate();
- if (!child_valid.ok()) {
- std::stringstream ss;
- ss << "Child array invalid: " << child_valid.ToString() << " at position {" << idx
- << "}";
- return Status::Invalid(ss.str());
- }
- ++idx;
- }
-
- if (array_length > 0 && array_length != length_) {
- return Status::Invalid("Struct's length is not equal to its child arrays");
- }
- }
- return Status::OK();
-}
-
-Status StructArray::Accept(ArrayVisitor* visitor) const {
- return visitor->Visit(*this);
-}
-
-Status StructBuilder::Finish(std::shared_ptr<Array>* out) {
- std::vector<std::shared_ptr<Array>> fields(field_builders_.size());
- for (size_t i = 0; i < field_builders_.size(); ++i) {
- RETURN_NOT_OK(field_builders_[i]->Finish(&fields[i]));
- }
-
- *out = std::make_shared<StructArray>(type_, length_, fields, null_count_, null_bitmap_);
-
- null_bitmap_ = nullptr;
- capacity_ = length_ = null_count_ = 0;
-
- return Status::OK();
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/struct.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h
deleted file mode 100644
index 1e2bf2d..0000000
--- a/cpp/src/arrow/types/struct.h
+++ /dev/null
@@ -1,116 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_STRUCT_H
-#define ARROW_TYPES_STRUCT_H
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-#include "arrow/types/list.h"
-#include "arrow/types/primitive.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class ARROW_EXPORT StructArray : public Array {
- public:
- using TypeClass = StructType;
-
- StructArray(const TypePtr& type, int32_t length, std::vector<ArrayPtr>& field_arrays,
- int32_t null_count = 0, std::shared_ptr<Buffer> null_bitmap = nullptr)
- : Array(type, length, null_count, null_bitmap) {
- type_ = type;
- field_arrays_ = field_arrays;
- }
-
- Status Validate() const override;
-
- virtual ~StructArray() {}
-
- // Return a shared pointer in case the requestor desires to share ownership
- // with this array.
- std::shared_ptr<Array> field(int32_t pos) const {
- DCHECK_GT(field_arrays_.size(), 0);
- return field_arrays_[pos];
- }
- const std::vector<ArrayPtr>& fields() const { return field_arrays_; }
-
- bool EqualsExact(const StructArray& other) const;
- bool Equals(const std::shared_ptr<Array>& arr) const override;
- bool RangeEquals(int32_t start_idx, int32_t end_idx, int32_t other_start_idx,
- const std::shared_ptr<Array>& arr) const override;
-
- Status Accept(ArrayVisitor* visitor) const override;
-
- protected:
- // The child arrays corresponding to each field of the struct data type.
- std::vector<ArrayPtr> field_arrays_;
-};
-
-// ---------------------------------------------------------------------------------
-// StructArray builder
-// Append, Resize and Reserve methods are acting on StructBuilder.
-// Please make sure all these methods of all child-builders' are consistently
-// called to maintain data-structure consistency.
-class ARROW_EXPORT StructBuilder : public ArrayBuilder {
- public:
- StructBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
- const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders)
- : ArrayBuilder(pool, type) {
- field_builders_ = field_builders;
- }
-
- Status Finish(std::shared_ptr<Array>* out) override;
-
- // Null bitmap is of equal length to every child field, and any zero byte
- // will be considered as a null for that field, but users must using app-
- // end methods or advance methods of the child builders' independently to
- // insert data.
- Status Append(int32_t length, const uint8_t* valid_bytes) {
- RETURN_NOT_OK(Reserve(length));
- UnsafeAppendToBitmap(valid_bytes, length);
- return Status::OK();
- }
-
- // Append an element to the Struct. All child-builders' Append method must
- // be called independently to maintain data-structure consistency.
- Status Append(bool is_valid = true) {
- RETURN_NOT_OK(Reserve(1));
- UnsafeAppendToBitmap(is_valid);
- return Status::OK();
- }
-
- Status AppendNull() { return Append(false); }
-
- std::shared_ptr<ArrayBuilder> field_builder(int pos) const {
- DCHECK_GT(field_builders_.size(), 0);
- return field_builders_[pos];
- }
- const std::vector<std::shared_ptr<ArrayBuilder>>& field_builders() const {
- return field_builders_;
- }
-
- protected:
- std::vector<std::shared_ptr<ArrayBuilder>> field_builders_;
-};
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_STRUCT_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/test-common.h b/cpp/src/arrow/types/test-common.h
deleted file mode 100644
index 6e6ab85..0000000
--- a/cpp/src/arrow/types/test-common.h
+++ /dev/null
@@ -1,70 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_TEST_COMMON_H
-#define ARROW_TYPES_TEST_COMMON_H
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-#include "arrow/util/memory-pool.h"
-
-namespace arrow {
-
-using std::unique_ptr;
-
-class TestBuilder : public ::testing::Test {
- public:
- void SetUp() {
- pool_ = default_memory_pool();
- type_ = TypePtr(new UInt8Type());
- builder_.reset(new UInt8Builder(pool_, type_));
- builder_nn_.reset(new UInt8Builder(pool_, type_));
- }
-
- protected:
- MemoryPool* pool_;
-
- TypePtr type_;
- unique_ptr<ArrayBuilder> builder_;
- unique_ptr<ArrayBuilder> builder_nn_;
-};
-
-template <class T, class Builder>
-Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
- int size, Builder* builder, ArrayPtr* out) {
- // Append the first 1000
- for (int i = 0; i < size; ++i) {
- if (valid_bytes[i] > 0) {
- RETURN_NOT_OK(builder->Append(values[i]));
- } else {
- RETURN_NOT_OK(builder->AppendNull());
- }
- }
- return builder->Finish(out);
-}
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_TEST_COMMON_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/union.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/union.cc b/cpp/src/arrow/types/union.cc
deleted file mode 100644
index cc2934b..0000000
--- a/cpp/src/arrow/types/union.cc
+++ /dev/null
@@ -1,27 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/types/union.h"
-
-#include <cstdlib>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "arrow/type.h"
-
-namespace arrow {} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/types/union.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/types/union.h b/cpp/src/arrow/types/union.h
deleted file mode 100644
index 44f39cc..0000000
--- a/cpp/src/arrow/types/union.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_TYPES_UNION_H
-#define ARROW_TYPES_UNION_H
-
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-class Buffer;
-
-class UnionArray : public Array {
- protected:
- // The data are types encoded as int16
- Buffer* types_;
- std::vector<std::shared_ptr<Array>> children_;
-};
-
-class DenseUnionArray : public UnionArray {
- protected:
- Buffer* offset_buf_;
-};
-
-class SparseUnionArray : public UnionArray {};
-
-} // namespace arrow
-
-#endif // ARROW_TYPES_UNION_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 6e19730..8d9afcc 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -22,12 +22,9 @@
# Headers: top level
install(FILES
bit-util.h
- buffer.h
logging.h
macros.h
- memory-pool.h
random.h
- status.h
visibility.h
DESTINATION include/arrow/util)
@@ -72,6 +69,3 @@ if (ARROW_BUILD_BENCHMARKS)
endif()
ADD_ARROW_TEST(bit-util-test)
-ADD_ARROW_TEST(buffer-test)
-ADD_ARROW_TEST(memory-pool-test)
-ADD_ARROW_TEST(status-test)
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/bit-util.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 7e1cb18..9c82407 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -18,9 +18,9 @@
#include <cstring>
#include <vector>
+#include "arrow/buffer.h"
+#include "arrow/status.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
namespace arrow {
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/buffer-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/buffer-test.cc b/cpp/src/arrow/util/buffer-test.cc
deleted file mode 100644
index 095b07b..0000000
--- a/cpp/src/arrow/util/buffer-test.cc
+++ /dev/null
@@ -1,140 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <limits>
-#include <string>
-
-#include "gtest/gtest.h"
-
-#include "arrow/test-util.h"
-#include "arrow/util/buffer.h"
-#include "arrow/util/status.h"
-
-using std::string;
-
-namespace arrow {
-
-class TestBuffer : public ::testing::Test {};
-
-TEST_F(TestBuffer, IsMutableFlag) {
- Buffer buf(nullptr, 0);
-
- ASSERT_FALSE(buf.is_mutable());
-
- MutableBuffer mbuf(nullptr, 0);
- ASSERT_TRUE(mbuf.is_mutable());
-
- PoolBuffer pbuf;
- ASSERT_TRUE(pbuf.is_mutable());
-}
-
-TEST_F(TestBuffer, Resize) {
- PoolBuffer buf;
-
- ASSERT_EQ(0, buf.size());
- ASSERT_OK(buf.Resize(100));
- ASSERT_EQ(100, buf.size());
- ASSERT_OK(buf.Resize(200));
- ASSERT_EQ(200, buf.size());
-
- // Make it smaller, too
- ASSERT_OK(buf.Resize(50));
- ASSERT_EQ(50, buf.size());
-}
-
-TEST_F(TestBuffer, ResizeOOM) {
- // realloc fails, even though there may be no explicit limit
- PoolBuffer buf;
- ASSERT_OK(buf.Resize(100));
- int64_t to_alloc = std::numeric_limits<int64_t>::max();
- ASSERT_RAISES(OutOfMemory, buf.Resize(to_alloc));
-}
-
-TEST_F(TestBuffer, EqualsWithSameContent) {
- MemoryPool* pool = default_memory_pool();
- const int32_t bufferSize = 128 * 1024;
- uint8_t* rawBuffer1;
- ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer1));
- memset(rawBuffer1, 12, bufferSize);
- uint8_t* rawBuffer2;
- ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer2));
- memset(rawBuffer2, 12, bufferSize);
- uint8_t* rawBuffer3;
- ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer3));
- memset(rawBuffer3, 3, bufferSize);
-
- Buffer buffer1(rawBuffer1, bufferSize);
- Buffer buffer2(rawBuffer2, bufferSize);
- Buffer buffer3(rawBuffer3, bufferSize);
- ASSERT_TRUE(buffer1.Equals(buffer2));
- ASSERT_FALSE(buffer1.Equals(buffer3));
-
- pool->Free(rawBuffer1, bufferSize);
- pool->Free(rawBuffer2, bufferSize);
- pool->Free(rawBuffer3, bufferSize);
-}
-
-TEST_F(TestBuffer, EqualsWithSameBuffer) {
- MemoryPool* pool = default_memory_pool();
- const int32_t bufferSize = 128 * 1024;
- uint8_t* rawBuffer;
- ASSERT_OK(pool->Allocate(bufferSize, &rawBuffer));
- memset(rawBuffer, 111, bufferSize);
-
- Buffer buffer1(rawBuffer, bufferSize);
- Buffer buffer2(rawBuffer, bufferSize);
- ASSERT_TRUE(buffer1.Equals(buffer2));
-
- const int64_t nbytes = bufferSize / 2;
- Buffer buffer3(rawBuffer, nbytes);
- ASSERT_TRUE(buffer1.Equals(buffer3, nbytes));
- ASSERT_FALSE(buffer1.Equals(buffer3, nbytes + 1));
-
- pool->Free(rawBuffer, bufferSize);
-}
-
-TEST_F(TestBuffer, Copy) {
- std::string data_str = "some data to copy";
-
- auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
-
- Buffer buf(data, data_str.size());
-
- std::shared_ptr<Buffer> out;
-
- ASSERT_OK(buf.Copy(5, 4, &out));
-
- Buffer expected(data + 5, 4);
- ASSERT_TRUE(out->Equals(expected));
-}
-
-TEST_F(TestBuffer, SliceBuffer) {
- std::string data_str = "some data to slice";
-
- auto data = reinterpret_cast<const uint8_t*>(data_str.c_str());
-
- auto buf = std::make_shared<Buffer>(data, data_str.size());
-
- std::shared_ptr<Buffer> out = SliceBuffer(buf, 5, 4);
- Buffer expected(data + 5, 4);
- ASSERT_TRUE(out->Equals(expected));
-
- ASSERT_EQ(2, buf.use_count());
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/buffer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/buffer.cc b/cpp/src/arrow/util/buffer.cc
deleted file mode 100644
index a230259..0000000
--- a/cpp/src/arrow/util/buffer.cc
+++ /dev/null
@@ -1,102 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/util/buffer.h"
-
-#include <cstdint>
-#include <limits>
-
-#include "arrow/util/bit-util.h"
-#include "arrow/util/logging.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-
-Buffer::Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size) {
- data_ = parent->data() + offset;
- size_ = size;
- parent_ = parent;
- capacity_ = size;
-}
-
-Buffer::~Buffer() {}
-
-Status Buffer::Copy(
- int64_t start, int64_t nbytes, MemoryPool* pool, std::shared_ptr<Buffer>* out) const {
- // Sanity checks
- DCHECK_LT(start, size_);
- DCHECK_LE(nbytes, size_ - start);
-
- auto new_buffer = std::make_shared<PoolBuffer>(pool);
- RETURN_NOT_OK(new_buffer->Resize(nbytes));
-
- std::memcpy(new_buffer->mutable_data(), data() + start, nbytes);
-
- *out = new_buffer;
- return Status::OK();
-}
-
-Status Buffer::Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const {
- return Copy(start, nbytes, default_memory_pool(), out);
-}
-
-std::shared_ptr<Buffer> SliceBuffer(
- const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length) {
- DCHECK_LT(offset, buffer->size());
- DCHECK_LE(length, buffer->size() - offset);
- return std::make_shared<Buffer>(buffer, offset, length);
-}
-
-std::shared_ptr<Buffer> MutableBuffer::GetImmutableView() {
- return std::make_shared<Buffer>(this->get_shared_ptr(), 0, size());
-}
-
-PoolBuffer::PoolBuffer(MemoryPool* pool) : ResizableBuffer(nullptr, 0) {
- if (pool == nullptr) { pool = default_memory_pool(); }
- pool_ = pool;
-}
-
-PoolBuffer::~PoolBuffer() {
- if (mutable_data_ != nullptr) { pool_->Free(mutable_data_, capacity_); }
-}
-
-Status PoolBuffer::Reserve(int64_t new_capacity) {
- if (!mutable_data_ || new_capacity > capacity_) {
- uint8_t* new_data;
- new_capacity = BitUtil::RoundUpToMultipleOf64(new_capacity);
- if (mutable_data_) {
- RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
- memcpy(new_data, mutable_data_, size_);
- pool_->Free(mutable_data_, capacity_);
- } else {
- RETURN_NOT_OK(pool_->Allocate(new_capacity, &new_data));
- }
- mutable_data_ = new_data;
- data_ = mutable_data_;
- capacity_ = new_capacity;
- }
- return Status::OK();
-}
-
-Status PoolBuffer::Resize(int64_t new_size) {
- RETURN_NOT_OK(Reserve(new_size));
- size_ = new_size;
- return Status::OK();
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/buffer.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h
deleted file mode 100644
index 5c87395..0000000
--- a/cpp/src/arrow/util/buffer.h
+++ /dev/null
@@ -1,232 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_UTIL_BUFFER_H
-#define ARROW_UTIL_BUFFER_H
-
-#include <algorithm>
-#include <cstdint>
-#include <cstring>
-#include <memory>
-
-#include "arrow/util/macros.h"
-#include "arrow/util/status.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class MemoryPool;
-class Status;
-
-// ----------------------------------------------------------------------
-// Buffer classes
-
-// Immutable API for a chunk of bytes which may or may not be owned by the
-// class instance. Buffers have two related notions of length: size and
-// capacity. Size is the number of bytes that might have valid data.
-// Capacity is the number of bytes that where allocated for the buffer in
-// total.
-// The following invariant is always true: Size < Capacity
-class ARROW_EXPORT Buffer : public std::enable_shared_from_this<Buffer> {
- public:
- Buffer(const uint8_t* data, int64_t size)
- : is_mutable_(false), data_(data), size_(size), capacity_(size) {}
- virtual ~Buffer();
-
- // An offset into data that is owned by another buffer, but we want to be
- // able to retain a valid pointer to it even after other shared_ptr's to the
- // parent buffer have been destroyed
- //
- // This method makes no assertions about alignment or padding of the buffer but
- // in general we expected buffers to be aligned and padded to 64 bytes. In the future
- // we might add utility methods to help determine if a buffer satisfies this contract.
- Buffer(const std::shared_ptr<Buffer>& parent, int64_t offset, int64_t size);
-
- std::shared_ptr<Buffer> get_shared_ptr() { return shared_from_this(); }
-
- bool is_mutable() const { return is_mutable_; }
-
- // Return true if both buffers are the same size and contain the same bytes
- // up to the number of compared bytes
- bool Equals(const Buffer& other, int64_t nbytes) const {
- return this == &other ||
- (size_ >= nbytes && other.size_ >= nbytes &&
- (data_ == other.data_ || !memcmp(data_, other.data_, nbytes)));
- }
-
- bool Equals(const Buffer& other) const {
- return this == &other ||
- (size_ == other.size_ &&
- (data_ == other.data_ || !memcmp(data_, other.data_, size_)));
- }
-
- // Copy section of buffer into a new Buffer
- Status Copy(int64_t start, int64_t nbytes, MemoryPool* pool,
- std::shared_ptr<Buffer>* out) const;
-
- // Default memory pool
- Status Copy(int64_t start, int64_t nbytes, std::shared_ptr<Buffer>* out) const;
-
- int64_t capacity() const { return capacity_; }
- const uint8_t* data() const { return data_; }
-
- int64_t size() const { return size_; }
-
- std::shared_ptr<Buffer> parent() const { return parent_; }
-
- protected:
- bool is_mutable_;
- const uint8_t* data_;
- int64_t size_;
- int64_t capacity_;
-
- // nullptr by default, but may be set
- std::shared_ptr<Buffer> parent_;
-
- private:
- DISALLOW_COPY_AND_ASSIGN(Buffer);
-};
-
-// Construct a view on passed buffer at the indicated offset and length. This
-// function cannot fail and does not error checking (except in debug builds)
-ARROW_EXPORT std::shared_ptr<Buffer> SliceBuffer(
- const std::shared_ptr<Buffer>& buffer, int64_t offset, int64_t length);
-
-// A Buffer whose contents can be mutated. May or may not own its data.
-class ARROW_EXPORT MutableBuffer : public Buffer {
- public:
- MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) {
- is_mutable_ = true;
- mutable_data_ = data;
- }
-
- uint8_t* mutable_data() { return mutable_data_; }
-
- // Get a read-only view of this buffer
- std::shared_ptr<Buffer> GetImmutableView();
-
- protected:
- MutableBuffer() : Buffer(nullptr, 0), mutable_data_(nullptr) {}
-
- uint8_t* mutable_data_;
-};
-
-class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
- public:
- // Change buffer reported size to indicated size, allocating memory if
- // necessary. This will ensure that the capacity of the buffer is a multiple
- // of 64 bytes as defined in Layout.md.
- virtual Status Resize(int64_t new_size) = 0;
-
- // Ensure that buffer has enough memory allocated to fit the indicated
- // capacity (and meets the 64 byte padding requirement in Layout.md).
- // It does not change buffer's reported size.
- virtual Status Reserve(int64_t new_capacity) = 0;
-
- protected:
- ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
-};
-
-// A Buffer whose lifetime is tied to a particular MemoryPool
-class ARROW_EXPORT PoolBuffer : public ResizableBuffer {
- public:
- explicit PoolBuffer(MemoryPool* pool = nullptr);
- virtual ~PoolBuffer();
-
- Status Resize(int64_t new_size) override;
- Status Reserve(int64_t new_capacity) override;
-
- private:
- MemoryPool* pool_;
-};
-
-class ARROW_EXPORT BufferBuilder {
- public:
- explicit BufferBuilder(MemoryPool* pool)
- : pool_(pool), data_(nullptr), capacity_(0), size_(0) {}
-
- // Resizes the buffer to the nearest multiple of 64 bytes per Layout.md
- Status Resize(int32_t elements) {
- if (capacity_ == 0) { buffer_ = std::make_shared<PoolBuffer>(pool_); }
- RETURN_NOT_OK(buffer_->Resize(elements));
- capacity_ = buffer_->capacity();
- data_ = buffer_->mutable_data();
- return Status::OK();
- }
-
- Status Append(const uint8_t* data, int length) {
- if (capacity_ < length + size_) { RETURN_NOT_OK(Resize(length + size_)); }
- UnsafeAppend(data, length);
- return Status::OK();
- }
-
- template <typename T>
- Status Append(T arithmetic_value) {
- static_assert(std::is_arithmetic<T>::value,
- "Convenience buffer append only supports arithmetic types");
- return Append(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
- }
-
- template <typename T>
- Status Append(const T* arithmetic_values, int num_elements) {
- static_assert(std::is_arithmetic<T>::value,
- "Convenience buffer append only supports arithmetic types");
- return Append(
- reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
- }
-
- // Unsafe methods don't check existing size
- void UnsafeAppend(const uint8_t* data, int length) {
- memcpy(data_ + size_, data, length);
- size_ += length;
- }
-
- template <typename T>
- void UnsafeAppend(T arithmetic_value) {
- static_assert(std::is_arithmetic<T>::value,
- "Convenience buffer append only supports arithmetic types");
- UnsafeAppend(reinterpret_cast<uint8_t*>(&arithmetic_value), sizeof(T));
- }
-
- template <typename T>
- void UnsafeAppend(const T* arithmetic_values, int num_elements) {
- static_assert(std::is_arithmetic<T>::value,
- "Convenience buffer append only supports arithmetic types");
- UnsafeAppend(
- reinterpret_cast<const uint8_t*>(arithmetic_values), num_elements * sizeof(T));
- }
-
- std::shared_ptr<Buffer> Finish() {
- auto result = buffer_;
- buffer_ = nullptr;
- capacity_ = size_ = 0;
- return result;
- }
- int capacity() { return capacity_; }
- int length() { return size_; }
-
- private:
- std::shared_ptr<PoolBuffer> buffer_;
- MemoryPool* pool_;
- uint8_t* data_;
- int64_t capacity_;
- int64_t size_;
-};
-
-} // namespace arrow
-
-#endif // ARROW_UTIL_BUFFER_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/memory-pool-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc
deleted file mode 100644
index 5d60376..0000000
--- a/cpp/src/arrow/util/memory-pool-test.cc
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <limits>
-
-#include "gtest/gtest.h"
-
-#include "arrow/test-util.h"
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-
-TEST(DefaultMemoryPool, MemoryTracking) {
- MemoryPool* pool = default_memory_pool();
-
- uint8_t* data;
- ASSERT_OK(pool->Allocate(100, &data));
- EXPECT_EQ(static_cast<uint64_t>(0), reinterpret_cast<uint64_t>(data) % 64);
- ASSERT_EQ(100, pool->bytes_allocated());
-
- pool->Free(data, 100);
- ASSERT_EQ(0, pool->bytes_allocated());
-}
-
-TEST(DefaultMemoryPool, OOM) {
- MemoryPool* pool = default_memory_pool();
-
- uint8_t* data;
- int64_t to_alloc = std::numeric_limits<int64_t>::max();
- ASSERT_RAISES(OutOfMemory, pool->Allocate(to_alloc, &data));
-}
-
-// Death tests and valgrind are known to not play well 100% of the time. See
-// googletest documentation
-#ifndef ARROW_VALGRIND
-
-TEST(DefaultMemoryPoolDeathTest, FreeLargeMemory) {
- MemoryPool* pool = default_memory_pool();
-
- uint8_t* data;
- ASSERT_OK(pool->Allocate(100, &data));
-
-#ifndef NDEBUG
- EXPECT_EXIT(pool->Free(data, 120), ::testing::ExitedWithCode(1),
- ".*Check failed: \\(bytes_allocated_\\) >= \\(size\\)");
-#endif
-
- pool->Free(data, 100);
-}
-
-#endif // ARROW_VALGRIND
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/memory-pool.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/memory-pool.cc b/cpp/src/arrow/util/memory-pool.cc
deleted file mode 100644
index 9aa7066..0000000
--- a/cpp/src/arrow/util/memory-pool.cc
+++ /dev/null
@@ -1,111 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/util/memory-pool.h"
-
-#include <cstdlib>
-#include <mutex>
-#include <sstream>
-#include <stdlib.h>
-
-#include "arrow/util/logging.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-
-namespace {
-// Allocate memory according to the alignment requirements for Arrow
-// (as of May 2016 64 bytes)
-Status AllocateAligned(int64_t size, uint8_t** out) {
- // TODO(emkornfield) find something compatible with windows
- constexpr size_t kAlignment = 64;
-#ifdef _MSC_VER
- // Special code path for MSVC
- *out = reinterpret_cast<uint8_t*>(_aligned_malloc(size, kAlignment));
- if (!*out) {
- std::stringstream ss;
- ss << "malloc of size " << size << " failed";
- return Status::OutOfMemory(ss.str());
- }
-#else
- const int result = posix_memalign(reinterpret_cast<void**>(out), kAlignment, size);
- if (result == ENOMEM) {
- std::stringstream ss;
- ss << "malloc of size " << size << " failed";
- return Status::OutOfMemory(ss.str());
- }
-
- if (result == EINVAL) {
- std::stringstream ss;
- ss << "invalid alignment parameter: " << kAlignment;
- return Status::Invalid(ss.str());
- }
-#endif
- return Status::OK();
-}
-} // namespace
-
-MemoryPool::~MemoryPool() {}
-
-class InternalMemoryPool : public MemoryPool {
- public:
- InternalMemoryPool() : bytes_allocated_(0) {}
- virtual ~InternalMemoryPool();
-
- Status Allocate(int64_t size, uint8_t** out) override;
-
- void Free(uint8_t* buffer, int64_t size) override;
-
- int64_t bytes_allocated() const override;
-
- private:
- mutable std::mutex pool_lock_;
- int64_t bytes_allocated_;
-};
-
-Status InternalMemoryPool::Allocate(int64_t size, uint8_t** out) {
- std::lock_guard<std::mutex> guard(pool_lock_);
- RETURN_NOT_OK(AllocateAligned(size, out));
- bytes_allocated_ += size;
-
- return Status::OK();
-}
-
-int64_t InternalMemoryPool::bytes_allocated() const {
- std::lock_guard<std::mutex> guard(pool_lock_);
- return bytes_allocated_;
-}
-
-void InternalMemoryPool::Free(uint8_t* buffer, int64_t size) {
- std::lock_guard<std::mutex> guard(pool_lock_);
- DCHECK_GE(bytes_allocated_, size);
-#ifdef _MSC_VER
- _aligned_free(buffer);
-#else
- std::free(buffer);
-#endif
- bytes_allocated_ -= size;
-}
-
-InternalMemoryPool::~InternalMemoryPool() {}
-
-MemoryPool* default_memory_pool() {
- static InternalMemoryPool default_memory_pool_;
- return &default_memory_pool_;
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/memory-pool.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/util/memory-pool.h
deleted file mode 100644
index 4c1d699..0000000
--- a/cpp/src/arrow/util/memory-pool.h
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_UTIL_MEMORY_POOL_H
-#define ARROW_UTIL_MEMORY_POOL_H
-
-#include <cstdint>
-
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Status;
-
-class ARROW_EXPORT MemoryPool {
- public:
- virtual ~MemoryPool();
-
- virtual Status Allocate(int64_t size, uint8_t** out) = 0;
- virtual void Free(uint8_t* buffer, int64_t size) = 0;
-
- virtual int64_t bytes_allocated() const = 0;
-};
-
-ARROW_EXPORT MemoryPool* default_memory_pool();
-
-} // namespace arrow
-
-#endif // ARROW_UTIL_MEMORY_POOL_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/status-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/status-test.cc b/cpp/src/arrow/util/status-test.cc
deleted file mode 100644
index e0ff20f..0000000
--- a/cpp/src/arrow/util/status-test.cc
+++ /dev/null
@@ -1,38 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "gtest/gtest.h"
-
-#include "arrow/test-util.h"
-#include "arrow/util/status.h"
-
-namespace arrow {
-
-TEST(StatusTest, TestCodeAndMessage) {
- Status ok = Status::OK();
- ASSERT_EQ(StatusCode::OK, ok.code());
- Status file_error = Status::IOError("file error");
- ASSERT_EQ(StatusCode::IOError, file_error.code());
- ASSERT_EQ("file error", file_error.message());
-}
-
-TEST(StatusTest, TestToString) {
- Status file_error = Status::IOError("file error");
- ASSERT_EQ("IOError: file error", file_error.ToString());
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/status.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc
deleted file mode 100644
index 08e9ae3..0000000
--- a/cpp/src/arrow/util/status.cc
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation. It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-#include "arrow/util/status.h"
-
-#include <assert.h>
-
-namespace arrow {
-
-Status::Status(StatusCode code, const std::string& msg, int16_t posix_code) {
- assert(code != StatusCode::OK);
- const uint32_t size = msg.size();
- char* result = new char[size + 7];
- memcpy(result, &size, sizeof(size));
- result[4] = static_cast<char>(code);
- memcpy(result + 5, &posix_code, sizeof(posix_code));
- memcpy(result + 7, msg.c_str(), msg.size());
- state_ = result;
-}
-
-const char* Status::CopyState(const char* state) {
- uint32_t size;
- memcpy(&size, state, sizeof(size));
- char* result = new char[size + 7];
- memcpy(result, state, size + 7);
- return result;
-}
-
-std::string Status::CodeAsString() const {
- if (state_ == NULL) { return "OK"; }
-
- const char* type;
- switch (code()) {
- case StatusCode::OK:
- type = "OK";
- break;
- case StatusCode::OutOfMemory:
- type = "Out of memory";
- break;
- case StatusCode::KeyError:
- type = "Key error";
- break;
- case StatusCode::TypeError:
- type = "Type error";
- break;
- case StatusCode::Invalid:
- type = "Invalid";
- break;
- case StatusCode::IOError:
- type = "IOError";
- break;
- case StatusCode::UnknownError:
- type = "Unknown error";
- break;
- case StatusCode::NotImplemented:
- type = "NotImplemented";
- break;
- default:
- type = "Unknown";
- break;
- }
- return std::string(type);
-}
-
-std::string Status::ToString() const {
- std::string result(CodeAsString());
- if (state_ == NULL) { return result; }
-
- result.append(": ");
-
- uint32_t length;
- memcpy(&length, state_, sizeof(length));
- result.append(reinterpret_cast<const char*>(state_ + 7), length);
- return result;
-}
-
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/cpp/src/arrow/util/status.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h
deleted file mode 100644
index 05f5b74..0000000
--- a/cpp/src/arrow/util/status.h
+++ /dev/null
@@ -1,192 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation. It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-// Adapted from Kudu github.com/cloudera/kudu
-
-#ifndef ARROW_STATUS_H_
-#define ARROW_STATUS_H_
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-#include "arrow/util/visibility.h"
-
-// Return the given status if it is not OK.
-#define ARROW_RETURN_NOT_OK(s) \
- do { \
- ::arrow::Status _s = (s); \
- if (!_s.ok()) { return _s; } \
- } while (0);
-
-// Return the given status if it is not OK, but first clone it and
-// prepend the given message.
-#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) \
- do { \
- ::arrow::Status _s = (s); \
- if (::gutil::PREDICT_FALSE(!_s.ok())) return _s.CloneAndPrepend(msg); \
- } while (0);
-
-// Return 'to_return' if 'to_call' returns a bad status.
-// The substitution for 'to_return' may reference the variable
-// 's' for the bad status.
-#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) \
- do { \
- ::arrow::Status s = (to_call); \
- if (::gutil::PREDICT_FALSE(!s.ok())) return (to_return); \
- } while (0);
-
-// If 'to_call' returns a bad status, CHECK immediately with a logged message
-// of 'msg' followed by the status.
-#define ARROW_CHECK_OK_PREPEND(to_call, msg) \
- do { \
- ::arrow::Status _s = (to_call); \
- ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \
- } while (0);
-
-// If the status is bad, CHECK immediately, appending the status to the
-// logged message.
-#define ARROW_CHECK_OK(s) ARROW_CHECK_OK_PREPEND(s, "Bad status")
-
-namespace arrow {
-
-#define RETURN_NOT_OK(s) \
- do { \
- Status _s = (s); \
- if (!_s.ok()) { return _s; } \
- } while (0);
-
-#define RETURN_NOT_OK_ELSE(s, else_) \
- do { \
- Status _s = (s); \
- if (!_s.ok()) { \
- else_; \
- return _s; \
- } \
- } while (0);
-
-enum class StatusCode : char {
- OK = 0,
- OutOfMemory = 1,
- KeyError = 2,
- TypeError = 3,
- Invalid = 4,
- IOError = 5,
- UnknownError = 9,
- NotImplemented = 10,
-};
-
-class ARROW_EXPORT Status {
- public:
- // Create a success status.
- Status() : state_(NULL) {}
- ~Status() { delete[] state_; }
-
- // Copy the specified status.
- Status(const Status& s);
- void operator=(const Status& s);
-
- // Return a success status.
- static Status OK() { return Status(); }
-
- // Return error status of an appropriate type.
- static Status OutOfMemory(const std::string& msg, int16_t posix_code = -1) {
- return Status(StatusCode::OutOfMemory, msg, posix_code);
- }
-
- static Status KeyError(const std::string& msg) {
- return Status(StatusCode::KeyError, msg, -1);
- }
-
- static Status TypeError(const std::string& msg) {
- return Status(StatusCode::TypeError, msg, -1);
- }
-
- static Status UnknownError(const std::string& msg) {
- return Status(StatusCode::UnknownError, msg, -1);
- }
-
- static Status NotImplemented(const std::string& msg) {
- return Status(StatusCode::NotImplemented, msg, -1);
- }
-
- static Status Invalid(const std::string& msg) {
- return Status(StatusCode::Invalid, msg, -1);
- }
-
- static Status IOError(const std::string& msg) {
- return Status(StatusCode::IOError, msg, -1);
- }
-
- // Returns true iff the status indicates success.
- bool ok() const { return (state_ == NULL); }
-
- bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
- bool IsKeyError() const { return code() == StatusCode::KeyError; }
- bool IsInvalid() const { return code() == StatusCode::Invalid; }
- bool IsIOError() const { return code() == StatusCode::IOError; }
-
- bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
- bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
-
- // Return a string representation of this status suitable for printing.
- // Returns the string "OK" for success.
- std::string ToString() const;
-
- // Return a string representation of the status code, without the message
- // text or posix code information.
- std::string CodeAsString() const;
-
- // Get the POSIX code associated with this Status, or -1 if there is none.
- int16_t posix_code() const;
-
- StatusCode code() const {
- return ((state_ == NULL) ? StatusCode::OK : static_cast<StatusCode>(state_[4]));
- }
-
- std::string message() const {
- uint32_t length;
- memcpy(&length, state_, sizeof(length));
- std::string msg;
- msg.append((state_ + 7), length);
- return msg;
- }
-
- private:
- // OK status has a NULL state_. Otherwise, state_ is a new[] array
- // of the following form:
- // state_[0..3] == length of message
- // state_[4] == code
- // state_[5..6] == posix_code
- // state_[7..] == message
- const char* state_;
-
- Status(StatusCode code, const std::string& msg, int16_t posix_code);
- static const char* CopyState(const char* s);
-};
-
-inline Status::Status(const Status& s) {
- state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-}
-
-inline void Status::operator=(const Status& s) {
- // The following condition catches both aliasing (when this == &s),
- // and the common case where both s and *this are ok.
- if (state_ != s.state_) {
- delete[] state_;
- state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
- }
-}
-
-} // namespace arrow
-
-#endif // ARROW_STATUS_H_
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/python/src/pyarrow/adapters/builtin.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/adapters/builtin.cc b/python/src/pyarrow/adapters/builtin.cc
index c034fbd..ac2f533 100644
--- a/python/src/pyarrow/adapters/builtin.cc
+++ b/python/src/pyarrow/adapters/builtin.cc
@@ -21,7 +21,7 @@
#include "pyarrow/adapters/builtin.h"
#include "arrow/api.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
#include "pyarrow/helpers.h"
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/python/src/pyarrow/adapters/pandas.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/adapters/pandas.cc b/python/src/pyarrow/adapters/pandas.cc
index adb27e8..64b7086 100644
--- a/python/src/pyarrow/adapters/pandas.cc
+++ b/python/src/pyarrow/adapters/pandas.cc
@@ -31,7 +31,7 @@
#include "arrow/api.h"
#include "arrow/util/bit-util.h"
-#include "arrow/util/status.h"
+#include "arrow/status.h"
#include "pyarrow/common.h"
#include "pyarrow/config.h"
http://git-wip-us.apache.org/repos/asf/arrow/blob/2c10d7cc/python/src/pyarrow/common.cc
----------------------------------------------------------------------
diff --git a/python/src/pyarrow/common.cc b/python/src/pyarrow/common.cc
index fa875f2..fb4d349 100644
--- a/python/src/pyarrow/common.cc
+++ b/python/src/pyarrow/common.cc
@@ -21,8 +21,8 @@
#include <mutex>
#include <sstream>
-#include "arrow/util/memory-pool.h"
-#include "arrow/util/status.h"
+#include "arrow/memory_pool.h"
+#include "arrow/status.h"
using arrow::Status;