You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/03/20 09:48:40 UTC
[1/3] arrow git commit: ARROW-661: [C++] Add LargeRecordBatch
metadata type, IPC support, associated refactoring
Repository: arrow
Updated Branches:
refs/heads/master 4c5f79c39 -> df2220f35
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/writer.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 7aff71e..1271652 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -45,6 +45,40 @@ class OutputStream;
namespace ipc {
+// Write the RecordBatch (collection of equal-length Arrow arrays) to the
+// output stream in a contiguous block. The record batch metadata is written as
+// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
+// prefixed by its size, followed by each of the memory buffers in the batch
+// written end to end (with appropriate alignment and padding):
+//
+// <int32: metadata size> <uint8*: metadata> <buffers>
+//
+// Finally, the absolute offsets (relative to the start of the output stream)
+// to the end of the body and end of the metadata / data header (suffixed by
+// the header size) is returned in out-variables
+//
+// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
+// default should be 0
+//
+// @param(out) metadata_length: the size of the length-prefixed flatbuffer
+// including padding to a 64-byte boundary
+//
+// @param(out) body_length: the size of the contiguous buffer block plus
+// padding bytes
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+ io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+ MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
+
+// Write Array as a DictionaryBatch message
+Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+ int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+ int64_t* body_length, MemoryPool* pool);
+
+// Compute the precise number of bytes needed in a contiguous memory segment to
+// write the record batch. This involves generating the complete serialized
+// Flatbuffers metadata.
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
+
class ARROW_EXPORT StreamWriter {
public:
virtual ~StreamWriter() = default;
@@ -68,10 +102,6 @@ class ARROW_EXPORT StreamWriter {
std::unique_ptr<StreamWriterImpl> impl_;
};
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
- const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
- io::OutputStream* out);
-
class ARROW_EXPORT FileWriter : public StreamWriter {
public:
static Status Open(io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
@@ -86,6 +116,14 @@ class ARROW_EXPORT FileWriter : public StreamWriter {
std::unique_ptr<FileWriterImpl> impl_;
};
+// ----------------------------------------------------------------------
+
+/// EXPERIMENTAL: Write record batch using LargeRecordBatch IPC metadata. This
+/// data may not be readable by all Arrow implementations
+Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+ io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+ MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
+
} // namespace ipc
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/loader.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/loader.h b/cpp/src/arrow/loader.h
index f116d64..9b650e2 100644
--- a/cpp/src/arrow/loader.h
+++ b/cpp/src/arrow/loader.h
@@ -41,11 +41,36 @@ struct DataType;
constexpr int kMaxNestingDepth = 64;
struct ARROW_EXPORT FieldMetadata {
+ FieldMetadata() {}
+ FieldMetadata(int64_t length, int64_t null_count, int64_t offset)
+ : length(length), null_count(null_count), offset(offset) {}
+
+ FieldMetadata(const FieldMetadata& other) {
+ this->length = other.length;
+ this->null_count = other.null_count;
+ this->offset = other.offset;
+ }
+
int64_t length;
int64_t null_count;
int64_t offset;
};
+struct ARROW_EXPORT BufferMetadata {
+ BufferMetadata() {}
+ BufferMetadata(int32_t page, int64_t offset, int64_t length)
+ : page(page), offset(offset), length(length) {}
+
+ /// The shared memory page id where to find this. Set to -1 if unused
+ int32_t page;
+
+ /// The relative offset into the memory page to the starting byte of the buffer
+ int64_t offset;
+
+ /// Absolute length in bytes of the buffer
+ int64_t length;
+};
+
/// Implement this to create new types of Arrow data loaders
class ARROW_EXPORT ArrayComponentSource {
public:
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/type.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index a143d79..adc3161 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -222,6 +222,7 @@ struct ARROW_EXPORT Field {
std::string ToString() const;
};
+
typedef std::shared_ptr<Field> FieldPtr;
struct ARROW_EXPORT PrimitiveCType : public FixedWidthType {
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/util/bit-util.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 3767ba9..ba0bfd7 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -112,7 +112,21 @@ Status CopyBitmap(MemoryPool* pool, const uint8_t* data, int64_t offset, int64_t
bool BitmapEquals(const uint8_t* left, int64_t left_offset, const uint8_t* right,
int64_t right_offset, int64_t bit_length) {
- // TODO(wesm): Make this faster using word-wise comparisons
+ if (left_offset % 8 == 0 && right_offset % 8 == 0) {
+ // byte aligned, can use memcmp
+ bool bytes_equal = std::memcmp(left + left_offset / 8, right + right_offset / 8,
+ bit_length / 8) == 0;
+ if (!bytes_equal) { return false; }
+ for (int64_t i = (bit_length / 8) * 8; i < bit_length; ++i) {
+ if (BitUtil::GetBit(left, left_offset + i) !=
+ BitUtil::GetBit(right, right_offset + i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // Unaligned slow case
for (int64_t i = 0; i < bit_length; ++i) {
if (BitUtil::GetBit(left, left_offset + i) !=
BitUtil::GetBit(right, right_offset + i)) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/format/Message.fbs
----------------------------------------------------------------------
diff --git a/format/Message.fbs b/format/Message.fbs
index 8fdcc80..2af26d4 100644
--- a/format/Message.fbs
+++ b/format/Message.fbs
@@ -308,6 +308,22 @@ table RecordBatch {
}
/// ----------------------------------------------------------------------
+/// EXPERIMENTAL: A RecordBatch type that supports data with more than 2^31 - 1
+/// elements. Arrow implementations do not need to implement this type to be
+/// compliant
+
+struct LargeFieldNode {
+ length: long;
+ null_count: long;
+}
+
+table LargeRecordBatch {
+ length: long;
+ nodes: [LargeFieldNode];
+ buffers: [Buffer];
+}
+
+/// ----------------------------------------------------------------------
/// For sending dictionary encoding information. Any Field can be
/// dictionary-encoded, but in this case none of its children may be
/// dictionary-encoded.
@@ -324,8 +340,12 @@ table DictionaryBatch {
/// This union enables us to easily send different message types without
/// redundant storage, and in the future we can easily add new message types.
+///
+/// Arrow implementations do not need to implement all of the message types,
+/// which may include experimental metadata types. For maximum compatibility,
+/// it is best to send data using RecordBatch
union MessageHeader {
- Schema, DictionaryBatch, RecordBatch
+ Schema, DictionaryBatch, RecordBatch, LargeRecordBatch
}
table Message {
[3/3] arrow git commit: ARROW-661: [C++] Add LargeRecordBatch
metadata type, IPC support, associated refactoring
Posted by uw...@apache.org.
ARROW-661: [C++] Add LargeRecordBatch metadata type, IPC support, associated refactoring
This patch enables the following code for writing record batches exceeding 2^31 - 1
```c++
RETURN_NOT_OK(WriteLargeRecordBatch(
batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
return ReadLargeRecordBatch(batch.schema(), 0, mmap_.get(), result);
```
This also does a fair amount of refactoring and code consolidation related to ongoing code cleaning in arrow_ipc.
These APIs are marked experimental. This does add `LargeRecordBatch` flatbuffer type to the Message union, but I've indicated that Arrow implementations (e.g. Java) are not required to implement this type. It's strictly to enable C++ users to write very large datasets that have been embedded for convenience in Arrow's structured data model.
cc @pcmoritz @robertnishihara
Author: Wes McKinney <we...@twosigma.com>
Closes #404 from wesm/ARROW-661 and squashes the following commits:
9c18a95 [Wes McKinney] Fix import ordering
d7811f2 [Wes McKinney] cpplint
179a1e3 [Wes McKinney] Add unit test for large record batches. Use bytewise comparisons with aligned bitmaps
36c3862 [Wes McKinney] Get LargeRecordBatch round trip working. Add to Message union for now
4c1d08c [Wes McKinney] Refactoring, failing test fixture for large record batch
f4c8830 [Wes McKinney] Consolidate ipc-metadata-test and ipc-read-write-test and draft large record batch read/write path
85d1a1c [Wes McKinney] Add (untested) metadata writer for LargeRecordBatch
0f2722c [Wes McKinney] Consolidate metadata-internal.h into metadata.h. Use own Arrow structs for IPC metadata and convert to flatbuffers later
e8f8973 [Wes McKinney] Split adapter.h/cc into reader.h/writer.h. Draft LargeRecordBatch type
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/df2220f3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/df2220f3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/df2220f3
Branch: refs/heads/master
Commit: df2220f350282925a454ed911eed6618e4d53969
Parents: 4c5f79c
Author: Wes McKinney <we...@twosigma.com>
Authored: Mon Mar 20 10:48:34 2017 +0100
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Mon Mar 20 10:48:34 2017 +0100
----------------------------------------------------------------------
cpp/src/arrow/allocator-test.cc | 1 +
cpp/src/arrow/allocator.h | 1 +
cpp/src/arrow/io/test-common.h | 18 +
cpp/src/arrow/ipc/CMakeLists.txt | 15 +-
cpp/src/arrow/ipc/adapter.cc | 630 -----------------------
cpp/src/arrow/ipc/adapter.h | 104 ----
cpp/src/arrow/ipc/api.h | 1 -
cpp/src/arrow/ipc/ipc-adapter-test.cc | 320 ------------
cpp/src/arrow/ipc/ipc-file-test.cc | 228 ---------
cpp/src/arrow/ipc/ipc-metadata-test.cc | 100 ----
cpp/src/arrow/ipc/ipc-read-write-test.cc | 608 ++++++++++++++++++++++
cpp/src/arrow/ipc/metadata-internal.cc | 597 ----------------------
cpp/src/arrow/ipc/metadata-internal.h | 83 ---
cpp/src/arrow/ipc/metadata.cc | 692 +++++++++++++++++++++++++-
cpp/src/arrow/ipc/metadata.h | 40 +-
cpp/src/arrow/ipc/reader.cc | 171 ++++++-
cpp/src/arrow/ipc/reader.h | 22 +
cpp/src/arrow/ipc/test-common.h | 2 +-
cpp/src/arrow/ipc/writer.cc | 544 ++++++++++++++++++--
cpp/src/arrow/ipc/writer.h | 46 +-
cpp/src/arrow/loader.h | 25 +
cpp/src/arrow/type.h | 1 +
cpp/src/arrow/util/bit-util.cc | 16 +-
format/Message.fbs | 22 +-
24 files changed, 2131 insertions(+), 2156 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/allocator-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/allocator-test.cc b/cpp/src/arrow/allocator-test.cc
index 0b24267..811ef5a 100644
--- a/cpp/src/arrow/allocator-test.cc
+++ b/cpp/src/arrow/allocator-test.cc
@@ -16,6 +16,7 @@
// under the License.
#include "gtest/gtest.h"
+
#include "arrow/allocator.h"
#include "arrow/test-util.h"
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/allocator.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/allocator.h b/cpp/src/arrow/allocator.h
index c976ba9..e00023d 100644
--- a/cpp/src/arrow/allocator.h
+++ b/cpp/src/arrow/allocator.h
@@ -21,6 +21,7 @@
#include <cstddef>
#include <memory>
#include <utility>
+
#include "arrow/memory_pool.h"
#include "arrow/status.h"
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/io/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/test-common.h b/cpp/src/arrow/io/test-common.h
index 8355714..4c11476 100644
--- a/cpp/src/arrow/io/test-common.h
+++ b/cpp/src/arrow/io/test-common.h
@@ -41,6 +41,24 @@
namespace arrow {
namespace io {
+static inline Status ZeroMemoryMap(MemoryMappedFile* file) {
+ constexpr int64_t kBufferSize = 512;
+ static constexpr uint8_t kZeroBytes[kBufferSize] = {0};
+
+ RETURN_NOT_OK(file->Seek(0));
+ int64_t position = 0;
+ int64_t file_size;
+ RETURN_NOT_OK(file->GetSize(&file_size));
+
+ int64_t chunksize;
+ while (position < file_size) {
+ chunksize = std::min(kBufferSize, file_size - position);
+ RETURN_NOT_OK(file->Write(kZeroBytes, chunksize));
+ position += chunksize;
+ }
+ return Status::OK();
+}
+
class MemoryMapFixture {
public:
void TearDown() {
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt
index c73af63..5d470df 100644
--- a/cpp/src/arrow/ipc/CMakeLists.txt
+++ b/cpp/src/arrow/ipc/CMakeLists.txt
@@ -29,12 +29,10 @@ set(ARROW_IPC_TEST_LINK_LIBS
arrow_io_static)
set(ARROW_IPC_SRCS
- adapter.cc
feather.cc
json.cc
json-internal.cc
metadata.cc
- metadata-internal.cc
reader.cc
writer.cc
)
@@ -64,16 +62,8 @@ ADD_ARROW_TEST(feather-test)
ARROW_TEST_LINK_LIBRARIES(feather-test
${ARROW_IPC_TEST_LINK_LIBS})
-ADD_ARROW_TEST(ipc-adapter-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-adapter-test
- ${ARROW_IPC_TEST_LINK_LIBS})
-
-ADD_ARROW_TEST(ipc-file-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-file-test
- ${ARROW_IPC_TEST_LINK_LIBS})
-
-ADD_ARROW_TEST(ipc-metadata-test)
-ARROW_TEST_LINK_LIBRARIES(ipc-metadata-test
+ADD_ARROW_TEST(ipc-read-write-test)
+ARROW_TEST_LINK_LIBRARIES(ipc-read-write-test
${ARROW_IPC_TEST_LINK_LIBS})
ADD_ARROW_TEST(ipc-json-test)
@@ -148,7 +138,6 @@ add_dependencies(arrow_ipc_objlib metadata_fbs)
# Headers: top level
install(FILES
- adapter.h
api.h
feather.h
json.h
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/adapter.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/adapter.cc b/cpp/src/arrow/ipc/adapter.cc
deleted file mode 100644
index db9f63c..0000000
--- a/cpp/src/arrow/ipc/adapter.cc
+++ /dev/null
@@ -1,630 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/ipc/adapter.h"
-
-#include <algorithm>
-#include <cstdint>
-#include <cstring>
-#include <limits>
-#include <sstream>
-#include <vector>
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/io/interfaces.h"
-#include "arrow/io/memory.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata-internal.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/util.h"
-#include "arrow/loader.h"
-#include "arrow/memory_pool.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/table.h"
-#include "arrow/type.h"
-#include "arrow/type_fwd.h"
-#include "arrow/util/bit-util.h"
-#include "arrow/util/logging.h"
-
-namespace arrow {
-
-namespace flatbuf = org::apache::arrow::flatbuf;
-
-namespace ipc {
-
-// ----------------------------------------------------------------------
-// Record batch write path
-
-class RecordBatchWriter : public ArrayVisitor {
- public:
- RecordBatchWriter(
- MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth)
- : pool_(pool),
- max_recursion_depth_(max_recursion_depth),
- buffer_start_offset_(buffer_start_offset) {
- DCHECK_GT(max_recursion_depth, 0);
- }
-
- virtual ~RecordBatchWriter() = default;
-
- Status VisitArray(const Array& arr) {
- if (max_recursion_depth_ <= 0) {
- return Status::Invalid("Max recursion depth reached");
- }
-
- if (arr.length() > std::numeric_limits<int32_t>::max()) {
- return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
- }
-
- // push back all common elements
- field_nodes_.push_back(flatbuf::FieldNode(
- static_cast<int32_t>(arr.length()), static_cast<int32_t>(arr.null_count())));
- if (arr.null_count() > 0) {
- std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
-
- if (arr.offset() != 0) {
- // With a sliced array / non-zero offset, we must copy the bitmap
- RETURN_NOT_OK(
- CopyBitmap(pool_, bitmap->data(), arr.offset(), arr.length(), &bitmap));
- }
-
- buffers_.push_back(bitmap);
- } else {
- // Push a dummy zero-length buffer, not to be copied
- buffers_.push_back(std::make_shared<Buffer>(nullptr, 0));
- }
- return arr.Accept(this);
- }
-
- Status Assemble(const RecordBatch& batch, int64_t* body_length) {
- if (field_nodes_.size() > 0) {
- field_nodes_.clear();
- buffer_meta_.clear();
- buffers_.clear();
- }
-
- // Perform depth-first traversal of the row-batch
- for (int i = 0; i < batch.num_columns(); ++i) {
- RETURN_NOT_OK(VisitArray(*batch.column(i)));
- }
-
- // The position for the start of a buffer relative to the passed frame of
- // reference. May be 0 or some other position in an address space
- int64_t offset = buffer_start_offset_;
-
- // Construct the buffer metadata for the record batch header
- for (size_t i = 0; i < buffers_.size(); ++i) {
- const Buffer* buffer = buffers_[i].get();
- int64_t size = 0;
- int64_t padding = 0;
-
- // The buffer might be null if we are handling zero row lengths.
- if (buffer) {
- size = buffer->size();
- padding = BitUtil::RoundUpToMultipleOf64(size) - size;
- }
-
- // TODO(wesm): We currently have no notion of shared memory page id's,
- // but we've included it in the metadata IDL for when we have it in the
- // future. Use page = -1 for now
- //
- // Note that page ids are a bespoke notion for Arrow and not a feature we
- // are using from any OS-level shared memory. The thought is that systems
- // may (in the future) associate integer page id's with physical memory
- // pages (according to whatever is the desired shared memory mechanism)
- buffer_meta_.push_back(flatbuf::Buffer(-1, offset, size + padding));
- offset += size + padding;
- }
-
- *body_length = offset - buffer_start_offset_;
- DCHECK(BitUtil::IsMultipleOf64(*body_length));
-
- return Status::OK();
- }
-
- // Override this for writing dictionary metadata
- virtual Status WriteMetadataMessage(
- int32_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
- return WriteRecordBatchMessage(
- num_rows, body_length, field_nodes_, buffer_meta_, out);
- }
-
- Status WriteMetadata(int32_t num_rows, int64_t body_length, io::OutputStream* dst,
- int32_t* metadata_length) {
- // Now that we have computed the locations of all of the buffers in shared
- // memory, the data header can be converted to a flatbuffer and written out
- //
- // Note: The memory written here is prefixed by the size of the flatbuffer
- // itself as an int32_t.
- std::shared_ptr<Buffer> metadata_fb;
- RETURN_NOT_OK(WriteMetadataMessage(num_rows, body_length, &metadata_fb));
-
- // Need to write 4 bytes (metadata size), the metadata, plus padding to
- // end on an 8-byte offset
- int64_t start_offset;
- RETURN_NOT_OK(dst->Tell(&start_offset));
-
- int32_t padded_metadata_length = static_cast<int32_t>(metadata_fb->size()) + 4;
- const int32_t remainder =
- (padded_metadata_length + static_cast<int32_t>(start_offset)) % 8;
- if (remainder != 0) { padded_metadata_length += 8 - remainder; }
-
- // The returned metadata size includes the length prefix, the flatbuffer,
- // plus padding
- *metadata_length = padded_metadata_length;
-
- // Write the flatbuffer size prefix including padding
- int32_t flatbuffer_size = padded_metadata_length - 4;
- RETURN_NOT_OK(
- dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
-
- // Write the flatbuffer
- RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
-
- // Write any padding
- int32_t padding =
- padded_metadata_length - static_cast<int32_t>(metadata_fb->size()) - 4;
- if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
-
- return Status::OK();
- }
-
- Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
- int64_t* body_length) {
- RETURN_NOT_OK(Assemble(batch, body_length));
-
-#ifndef NDEBUG
- int64_t start_position, current_position;
- RETURN_NOT_OK(dst->Tell(&start_position));
-#endif
-
- RETURN_NOT_OK(WriteMetadata(
- static_cast<int32_t>(batch.num_rows()), *body_length, dst, metadata_length));
-
-#ifndef NDEBUG
- RETURN_NOT_OK(dst->Tell(¤t_position));
- DCHECK(BitUtil::IsMultipleOf8(current_position));
-#endif
-
- // Now write the buffers
- for (size_t i = 0; i < buffers_.size(); ++i) {
- const Buffer* buffer = buffers_[i].get();
- int64_t size = 0;
- int64_t padding = 0;
-
- // The buffer might be null if we are handling zero row lengths.
- if (buffer) {
- size = buffer->size();
- padding = BitUtil::RoundUpToMultipleOf64(size) - size;
- }
-
- if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
-
- if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
- }
-
-#ifndef NDEBUG
- RETURN_NOT_OK(dst->Tell(¤t_position));
- DCHECK(BitUtil::IsMultipleOf8(current_position));
-#endif
-
- return Status::OK();
- }
-
- Status GetTotalSize(const RecordBatch& batch, int64_t* size) {
- // emulates the behavior of Write without actually writing
- int32_t metadata_length = 0;
- int64_t body_length = 0;
- MockOutputStream dst;
- RETURN_NOT_OK(Write(batch, &dst, &metadata_length, &body_length));
- *size = dst.GetExtentBytesWritten();
- return Status::OK();
- }
-
- protected:
- template <typename ArrayType>
- Status VisitFixedWidth(const ArrayType& array) {
- std::shared_ptr<Buffer> data_buffer = array.data();
-
- if (array.offset() != 0) {
- // Non-zero offset, slice the buffer
- const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
- const int type_width = fw_type.bit_width() / 8;
- const int64_t byte_offset = array.offset() * type_width;
-
- // Send padding if it's available
- const int64_t buffer_length =
- std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
- data_buffer->size() - byte_offset);
- data_buffer = SliceBuffer(data_buffer, byte_offset, buffer_length);
- }
- buffers_.push_back(data_buffer);
- return Status::OK();
- }
-
- template <typename ArrayType>
- Status GetZeroBasedValueOffsets(
- const ArrayType& array, std::shared_ptr<Buffer>* value_offsets) {
- // Share slicing logic between ListArray and BinaryArray
-
- auto offsets = array.value_offsets();
-
- if (array.offset() != 0) {
- // If we have a non-zero offset, then the value offsets do not start at
- // zero. We must a) create a new offsets array with shifted offsets and
- // b) slice the values array accordingly
-
- std::shared_ptr<MutableBuffer> shifted_offsets;
- RETURN_NOT_OK(AllocateBuffer(
- pool_, sizeof(int32_t) * (array.length() + 1), &shifted_offsets));
-
- int32_t* dest_offsets = reinterpret_cast<int32_t*>(shifted_offsets->mutable_data());
- const int32_t start_offset = array.value_offset(0);
-
- for (int i = 0; i < array.length(); ++i) {
- dest_offsets[i] = array.value_offset(i) - start_offset;
- }
- // Final offset
- dest_offsets[array.length()] = array.value_offset(array.length()) - start_offset;
- offsets = shifted_offsets;
- }
-
- *value_offsets = offsets;
- return Status::OK();
- }
-
- Status VisitBinary(const BinaryArray& array) {
- std::shared_ptr<Buffer> value_offsets;
- RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
- auto data = array.data();
-
- if (array.offset() != 0) {
- // Slice the data buffer to include only the range we need now
- data = SliceBuffer(data, array.value_offset(0), array.value_offset(array.length()));
- }
-
- buffers_.push_back(value_offsets);
- buffers_.push_back(data);
- return Status::OK();
- }
-
- Status Visit(const FixedWidthBinaryArray& array) override {
- auto data = array.data();
- int32_t width = array.byte_width();
-
- if (array.offset() != 0) {
- data = SliceBuffer(data, array.offset() * width, width * array.length());
- }
- buffers_.push_back(data);
- return Status::OK();
- }
-
- Status Visit(const BooleanArray& array) override {
- buffers_.push_back(array.data());
- return Status::OK();
- }
-
-#define VISIT_FIXED_WIDTH(TYPE) \
- Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
-
- VISIT_FIXED_WIDTH(Int8Array);
- VISIT_FIXED_WIDTH(Int16Array);
- VISIT_FIXED_WIDTH(Int32Array);
- VISIT_FIXED_WIDTH(Int64Array);
- VISIT_FIXED_WIDTH(UInt8Array);
- VISIT_FIXED_WIDTH(UInt16Array);
- VISIT_FIXED_WIDTH(UInt32Array);
- VISIT_FIXED_WIDTH(UInt64Array);
- VISIT_FIXED_WIDTH(HalfFloatArray);
- VISIT_FIXED_WIDTH(FloatArray);
- VISIT_FIXED_WIDTH(DoubleArray);
- VISIT_FIXED_WIDTH(DateArray);
- VISIT_FIXED_WIDTH(Date32Array);
- VISIT_FIXED_WIDTH(TimeArray);
- VISIT_FIXED_WIDTH(TimestampArray);
-
-#undef VISIT_FIXED_WIDTH
-
- Status Visit(const StringArray& array) override { return VisitBinary(array); }
-
- Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
-
- Status Visit(const ListArray& array) override {
- std::shared_ptr<Buffer> value_offsets;
- RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
- buffers_.push_back(value_offsets);
-
- --max_recursion_depth_;
- std::shared_ptr<Array> values = array.values();
-
- if (array.offset() != 0) {
- // For non-zero offset, we slice the values array accordingly
- const int32_t offset = array.value_offset(0);
- const int32_t length = array.value_offset(array.length()) - offset;
- values = values->Slice(offset, length);
- }
- RETURN_NOT_OK(VisitArray(*values));
- ++max_recursion_depth_;
- return Status::OK();
- }
-
- Status Visit(const StructArray& array) override {
- --max_recursion_depth_;
- for (std::shared_ptr<Array> field : array.fields()) {
- if (array.offset() != 0) {
- // If offset is non-zero, slice the child array
- field = field->Slice(array.offset(), array.length());
- }
- RETURN_NOT_OK(VisitArray(*field));
- }
- ++max_recursion_depth_;
- return Status::OK();
- }
-
- Status Visit(const UnionArray& array) override {
- auto type_ids = array.type_ids();
- if (array.offset() != 0) {
- type_ids = SliceBuffer(type_ids, array.offset() * sizeof(UnionArray::type_id_t),
- array.length() * sizeof(UnionArray::type_id_t));
- }
-
- buffers_.push_back(type_ids);
-
- --max_recursion_depth_;
- if (array.mode() == UnionMode::DENSE) {
- const auto& type = static_cast<const UnionType&>(*array.type());
- auto value_offsets = array.value_offsets();
-
- // The Union type codes are not necessary 0-indexed
- uint8_t max_code = 0;
- for (uint8_t code : type.type_codes) {
- if (code > max_code) { max_code = code; }
- }
-
- // Allocate an array of child offsets. Set all to -1 to indicate that we
- // haven't observed a first occurrence of a particular child yet
- std::vector<int32_t> child_offsets(max_code + 1);
- std::vector<int32_t> child_lengths(max_code + 1, 0);
-
- if (array.offset() != 0) {
- // This is an unpleasant case. Because the offsets are different for
- // each child array, when we have a sliced array, we need to "rebase"
- // the value_offsets for each array
-
- const int32_t* unshifted_offsets = array.raw_value_offsets();
- const uint8_t* type_ids = array.raw_type_ids();
-
- // Allocate the shifted offsets
- std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
- RETURN_NOT_OK(AllocateBuffer(
- pool_, array.length() * sizeof(int32_t), &shifted_offsets_buffer));
- int32_t* shifted_offsets =
- reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
-
- for (int64_t i = 0; i < array.length(); ++i) {
- const uint8_t code = type_ids[i];
- int32_t shift = child_offsets[code];
- if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
- shifted_offsets[i] = unshifted_offsets[i] - shift;
-
- // Update the child length to account for observed value
- ++child_lengths[code];
- }
-
- value_offsets = shifted_offsets_buffer;
- }
- buffers_.push_back(value_offsets);
-
- // Visit children and slice accordingly
- for (int i = 0; i < type.num_children(); ++i) {
- std::shared_ptr<Array> child = array.child(i);
- if (array.offset() != 0) {
- const uint8_t code = type.type_codes[i];
- child = child->Slice(child_offsets[code], child_lengths[code]);
- }
- RETURN_NOT_OK(VisitArray(*child));
- }
- } else {
- for (std::shared_ptr<Array> child : array.children()) {
- // Sparse union, slicing is simpler
- if (array.offset() != 0) {
- // If offset is non-zero, slice the child array
- child = child->Slice(array.offset(), array.length());
- }
- RETURN_NOT_OK(VisitArray(*child));
- }
- }
- ++max_recursion_depth_;
- return Status::OK();
- }
-
- Status Visit(const DictionaryArray& array) override {
- // Dictionary written out separately. Slice offset contained in the indices
- return array.indices()->Accept(this);
- }
-
- // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
- MemoryPool* pool_;
-
- std::vector<flatbuf::FieldNode> field_nodes_;
- std::vector<flatbuf::Buffer> buffer_meta_;
- std::vector<std::shared_ptr<Buffer>> buffers_;
-
- int64_t max_recursion_depth_;
- int64_t buffer_start_offset_;
-};
-
-class DictionaryWriter : public RecordBatchWriter {
- public:
- using RecordBatchWriter::RecordBatchWriter;
-
- Status WriteMetadataMessage(
- int32_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
- return WriteDictionaryMessage(
- dictionary_id_, num_rows, body_length, field_nodes_, buffer_meta_, out);
- }
-
- Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
- io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
- dictionary_id_ = dictionary_id;
-
- // Make a dummy record batch. A bit tedious as we have to make a schema
- std::vector<std::shared_ptr<Field>> fields = {
- arrow::field("dictionary", dictionary->type())};
- auto schema = std::make_shared<Schema>(fields);
- RecordBatch batch(schema, dictionary->length(), {dictionary});
-
- return RecordBatchWriter::Write(batch, dst, metadata_length, body_length);
- }
-
- private:
- // TODO(wesm): Setting this in Write is a bit unclean, but it works
- int64_t dictionary_id_;
-};
-
-Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
- io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
- MemoryPool* pool, int max_recursion_depth) {
- RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
- return writer.Write(batch, dst, metadata_length, body_length);
-}
-
-Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
- int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
- int64_t* body_length, MemoryPool* pool) {
- DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth);
- return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
-}
-
-Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
- RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth);
- RETURN_NOT_OK(writer.GetTotalSize(batch, size));
- return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Record batch read path
-
-class IpcComponentSource : public ArrayComponentSource {
- public:
- IpcComponentSource(const RecordBatchMetadata& metadata, io::RandomAccessFile* file)
- : metadata_(metadata), file_(file) {}
-
- Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
- BufferMetadata buffer_meta = metadata_.buffer(buffer_index);
- if (buffer_meta.length == 0) {
- *out = nullptr;
- return Status::OK();
- } else {
- return file_->ReadAt(buffer_meta.offset, buffer_meta.length, out);
- }
- }
-
- Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
- // pop off a field
- if (field_index >= metadata_.num_fields()) {
- return Status::Invalid("Ran out of field metadata, likely malformed");
- }
- *metadata = metadata_.field(field_index);
- return Status::OK();
- }
-
- private:
- const RecordBatchMetadata& metadata_;
- io::RandomAccessFile* file_;
-};
-
-class RecordBatchReader {
- public:
- RecordBatchReader(const RecordBatchMetadata& metadata,
- const std::shared_ptr<Schema>& schema, int max_recursion_depth,
- io::RandomAccessFile* file)
- : metadata_(metadata),
- schema_(schema),
- max_recursion_depth_(max_recursion_depth),
- file_(file) {}
-
- Status Read(std::shared_ptr<RecordBatch>* out) {
- std::vector<std::shared_ptr<Array>> arrays(schema_->num_fields());
-
- IpcComponentSource source(metadata_, file_);
- ArrayLoaderContext context;
- context.source = &source;
- context.field_index = 0;
- context.buffer_index = 0;
- context.max_recursion_depth = max_recursion_depth_;
-
- for (int i = 0; i < schema_->num_fields(); ++i) {
- RETURN_NOT_OK(LoadArray(schema_->field(i)->type, &context, &arrays[i]));
- }
-
- *out = std::make_shared<RecordBatch>(schema_, metadata_.length(), arrays);
- return Status::OK();
- }
-
- private:
- const RecordBatchMetadata& metadata_;
- std::shared_ptr<Schema> schema_;
- int max_recursion_depth_;
- io::RandomAccessFile* file_;
-};
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
- const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
- std::shared_ptr<RecordBatch>* out) {
- return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
-}
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
- const std::shared_ptr<Schema>& schema, int max_recursion_depth,
- io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
- RecordBatchReader reader(metadata, schema, max_recursion_depth, file);
- return reader.Read(out);
-}
-
-Status ReadDictionary(const DictionaryBatchMetadata& metadata,
- const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
- std::shared_ptr<Array>* out) {
- int64_t id = metadata.id();
- auto it = dictionary_types.find(id);
- if (it == dictionary_types.end()) {
- std::stringstream ss;
- ss << "Do not have type metadata for dictionary with id: " << id;
- return Status::KeyError(ss.str());
- }
-
- std::vector<std::shared_ptr<Field>> fields = {it->second};
-
- // We need a schema for the record batch
- auto dummy_schema = std::make_shared<Schema>(fields);
-
- // The dictionary is embedded in a record batch with a single column
- std::shared_ptr<RecordBatch> batch;
- RETURN_NOT_OK(ReadRecordBatch(metadata.record_batch(), dummy_schema, file, &batch));
-
- if (batch->num_columns() != 1) {
- return Status::Invalid("Dictionary record batch must only contain one field");
- }
-
- *out = batch->column(0);
- return Status::OK();
-}
-
-} // namespace ipc
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/adapter.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/adapter.h b/cpp/src/arrow/ipc/adapter.h
deleted file mode 100644
index cea4686..0000000
--- a/cpp/src/arrow/ipc/adapter.h
+++ /dev/null
@@ -1,104 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Public API for writing and accessing (with zero copy, if possible) Arrow
-// IPC binary formatted data (e.g. in shared memory, or from some other IO source)
-
-#ifndef ARROW_IPC_ADAPTER_H
-#define ARROW_IPC_ADAPTER_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "arrow/ipc/metadata.h"
-#include "arrow/loader.h"
-#include "arrow/util/visibility.h"
-
-namespace arrow {
-
-class Array;
-class MemoryPool;
-class RecordBatch;
-class Schema;
-class Status;
-
-namespace io {
-
-class RandomAccessFile;
-class OutputStream;
-
-} // namespace io
-
-namespace ipc {
-
-// ----------------------------------------------------------------------
-// Write path
-
-// Write the RecordBatch (collection of equal-length Arrow arrays) to the
-// output stream in a contiguous block. The record batch metadata is written as
-// a flatbuffer (see format/Message.fbs -- the RecordBatch message type)
-// prefixed by its size, followed by each of the memory buffers in the batch
-// written end to end (with appropriate alignment and padding):
-//
-// <int32: metadata size> <uint8*: metadata> <buffers>
-//
-// Finally, the absolute offsets (relative to the start of the output stream)
-// to the end of the body and end of the metadata / data header (suffixed by
-// the header size) is returned in out-variables
-//
-// @param(in) buffer_start_offset: the start offset to use in the buffer metadata,
-// default should be 0
-//
-// @param(out) metadata_length: the size of the length-prefixed flatbuffer
-// including padding to a 64-byte boundary
-//
-// @param(out) body_length: the size of the contiguous buffer block plus
-// padding bytes
-Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
- io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
- MemoryPool* pool, int max_recursion_depth = kMaxNestingDepth);
-
-// Write Array as a DictionaryBatch message
-Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
- int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
- int64_t* body_length, MemoryPool* pool);
-
-// Compute the precise number of bytes needed in a contiguous memory segment to
-// write the record batch. This involves generating the complete serialized
-// Flatbuffers metadata.
-Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size);
-
-// ----------------------------------------------------------------------
-// "Read" path; does not copy data if the input supports zero copy reads
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
- const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
- std::shared_ptr<RecordBatch>* out);
-
-Status ReadRecordBatch(const RecordBatchMetadata& metadata,
- const std::shared_ptr<Schema>& schema, int max_recursion_depth,
- io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
-
-Status ReadDictionary(const DictionaryBatchMetadata& metadata,
- const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
- std::shared_ptr<Array>* out);
-
-} // namespace ipc
-} // namespace arrow
-
-#endif // ARROW_IPC_MEMORY_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/api.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/api.h b/cpp/src/arrow/ipc/api.h
index ad7cd84..3f05e69 100644
--- a/cpp/src/arrow/ipc/api.h
+++ b/cpp/src/arrow/ipc/api.h
@@ -18,7 +18,6 @@
#ifndef ARROW_IPC_API_H
#define ARROW_IPC_API_H
-#include "arrow/ipc/adapter.h"
#include "arrow/ipc/feather.h"
#include "arrow/ipc/json.h"
#include "arrow/ipc/metadata.h"
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/ipc-adapter-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-adapter-test.cc b/cpp/src/arrow/ipc/ipc-adapter-test.cc
deleted file mode 100644
index 638d98a..0000000
--- a/cpp/src/arrow/ipc/ipc-adapter-test.cc
+++ /dev/null
@@ -1,320 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdio>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/io/memory.h"
-#include "arrow/io/test-common.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/ipc/util.h"
-
-#include "arrow/buffer.h"
-#include "arrow/memory_pool.h"
-#include "arrow/pretty_print.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-#include "arrow/util/bit-util.h"
-
-namespace arrow {
-namespace ipc {
-
-class IpcTestFixture : public io::MemoryMapFixture {
- public:
- Status RoundTripHelper(const RecordBatch& batch, int memory_map_size,
- std::shared_ptr<RecordBatch>* batch_result) {
- std::string path = "test-write-row-batch";
- io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
-
- int32_t metadata_length;
- int64_t body_length;
-
- const int64_t buffer_offset = 0;
-
- RETURN_NOT_OK(WriteRecordBatch(
- batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
-
- std::shared_ptr<Message> message;
- RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
- auto metadata = std::make_shared<RecordBatchMetadata>(message);
-
- // The buffer offsets start at 0, so we must construct a
- // RandomAccessFile according to that frame of reference
- std::shared_ptr<Buffer> buffer_payload;
- RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
- io::BufferReader buffer_reader(buffer_payload);
-
- return ReadRecordBatch(*metadata, batch.schema(), &buffer_reader, batch_result);
- }
-
- void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
- std::shared_ptr<RecordBatch> batch_result;
-
- ASSERT_OK(RoundTripHelper(batch, 1 << 16, &batch_result));
- EXPECT_EQ(batch.num_rows(), batch_result->num_rows());
-
- ASSERT_TRUE(batch.schema()->Equals(batch_result->schema()));
- ASSERT_EQ(batch.num_columns(), batch_result->num_columns())
- << batch.schema()->ToString()
- << " result: " << batch_result->schema()->ToString();
-
- for (int i = 0; i < batch.num_columns(); ++i) {
- const auto& left = *batch.column(i);
- const auto& right = *batch_result->column(i);
- if (!left.Equals(right)) {
- std::stringstream pp_result;
- std::stringstream pp_expected;
-
- ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
- ASSERT_OK(PrettyPrint(right, 0, &pp_result));
-
- FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
- << "\nGot: " << pp_result.str();
- }
- }
- }
-
- void CheckRoundtrip(const std::shared_ptr<Array>& array, int64_t buffer_size) {
- auto f0 = arrow::field("f0", array->type());
- std::vector<std::shared_ptr<Field>> fields = {f0};
- auto schema = std::make_shared<Schema>(fields);
-
- RecordBatch batch(schema, 0, {array});
- CheckRoundtrip(batch, buffer_size);
- }
-
- protected:
- std::shared_ptr<io::MemoryMappedFile> mmap_;
- MemoryPool* pool_;
-};
-
-class TestWriteRecordBatch : public ::testing::Test, public IpcTestFixture {
- public:
- void SetUp() { pool_ = default_memory_pool(); }
- void TearDown() { io::MemoryMapFixture::TearDown(); }
-};
-
-class TestRecordBatchParam : public ::testing::TestWithParam<MakeRecordBatch*>,
- public IpcTestFixture {
- public:
- void SetUp() { pool_ = default_memory_pool(); }
- void TearDown() { io::MemoryMapFixture::TearDown(); }
- using IpcTestFixture::RoundTripHelper;
- using IpcTestFixture::CheckRoundtrip;
-};
-
-TEST_P(TestRecordBatchParam, RoundTrip) {
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
-
- CheckRoundtrip(*batch, 1 << 20);
-}
-
-TEST_P(TestRecordBatchParam, SliceRoundTrip) {
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
-
- // Skip the zero-length case
- if (batch->num_rows() < 2) { return; }
-
- auto sliced_batch = batch->Slice(2, 10);
- CheckRoundtrip(*sliced_batch, 1 << 20);
-}
-
-TEST_P(TestRecordBatchParam, ZeroLengthArrays) {
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
-
- std::shared_ptr<RecordBatch> zero_length_batch;
- if (batch->num_rows() > 2) {
- zero_length_batch = batch->Slice(2, 0);
- } else {
- zero_length_batch = batch->Slice(0, 0);
- }
-
- CheckRoundtrip(*zero_length_batch, 1 << 20);
-
- // ARROW-544: check binary array
- std::shared_ptr<MutableBuffer> value_offsets;
- ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
- *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
-
- std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(0, value_offsets,
- std::make_shared<Buffer>(nullptr, 0), std::make_shared<Buffer>(nullptr, 0));
-
- // null value_offsets
- std::shared_ptr<Array> bin_array2 = std::make_shared<BinaryArray>(0, nullptr, nullptr);
-
- CheckRoundtrip(bin_array, 1 << 20);
- CheckRoundtrip(bin_array2, 1 << 20);
-}
-
-INSTANTIATE_TEST_CASE_P(
- RoundTripTests, TestRecordBatchParam,
- ::testing::Values(&MakeIntRecordBatch, &MakeStringTypesRecordBatch,
- &MakeNonNullRecordBatch, &MakeZeroLengthRecordBatch, &MakeListRecordBatch,
- &MakeDeeplyNestedList, &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate,
- &MakeTimestamps, &MakeTimes, &MakeFWBinary));
-
-void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
- ipc::MockOutputStream mock;
- int32_t mock_metadata_length = -1;
- int64_t mock_body_length = -1;
- int64_t size = -1;
- ASSERT_OK(WriteRecordBatch(
- *batch, 0, &mock, &mock_metadata_length, &mock_body_length, default_memory_pool()));
- ASSERT_OK(GetRecordBatchSize(*batch, &size));
- ASSERT_EQ(mock.GetExtentBytesWritten(), size);
-}
-
-TEST_F(TestWriteRecordBatch, IntegerGetRecordBatchSize) {
- std::shared_ptr<RecordBatch> batch;
-
- ASSERT_OK(MakeIntRecordBatch(&batch));
- TestGetRecordBatchSize(batch);
-
- ASSERT_OK(MakeListRecordBatch(&batch));
- TestGetRecordBatchSize(batch);
-
- ASSERT_OK(MakeZeroLengthRecordBatch(&batch));
- TestGetRecordBatchSize(batch);
-
- ASSERT_OK(MakeNonNullRecordBatch(&batch));
- TestGetRecordBatchSize(batch);
-
- ASSERT_OK(MakeDeeplyNestedList(&batch));
- TestGetRecordBatchSize(batch);
-}
-
-class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
- public:
- void SetUp() { pool_ = default_memory_pool(); }
- void TearDown() { io::MemoryMapFixture::TearDown(); }
-
- Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
- int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
- std::shared_ptr<Schema>* schema) {
- const int batch_length = 5;
- TypePtr type = int32();
- std::shared_ptr<Array> array;
- const bool include_nulls = true;
- RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
- for (int i = 0; i < recursion_level; ++i) {
- type = list(type);
- RETURN_NOT_OK(
- MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
- }
-
- auto f0 = field("f0", type);
-
- *schema = std::shared_ptr<Schema>(new Schema({f0}));
-
- std::vector<std::shared_ptr<Array>> arrays = {array};
- *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
-
- std::string path = "test-write-past-max-recursion";
- const int memory_map_size = 1 << 20;
- io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
-
- if (override_level) {
- return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
- pool_, recursion_level + 1);
- } else {
- return WriteRecordBatch(
- **batch, 0, mmap_.get(), metadata_length, body_length, pool_);
- }
- }
-
- protected:
- std::shared_ptr<io::MemoryMappedFile> mmap_;
- MemoryPool* pool_;
-};
-
-TEST_F(RecursionLimits, WriteLimit) {
- int32_t metadata_length = -1;
- int64_t body_length = -1;
- std::shared_ptr<Schema> schema;
- std::shared_ptr<RecordBatch> batch;
- ASSERT_RAISES(Invalid,
- WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &batch, &schema));
-}
-
-TEST_F(RecursionLimits, ReadLimit) {
- int32_t metadata_length = -1;
- int64_t body_length = -1;
- std::shared_ptr<Schema> schema;
-
- const int recursion_depth = 64;
-
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK(WriteToMmap(
- recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
-
- std::shared_ptr<Message> message;
- ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
- auto metadata = std::make_shared<RecordBatchMetadata>(message);
-
- std::shared_ptr<Buffer> payload;
- ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
-
- io::BufferReader reader(payload);
-
- std::shared_ptr<RecordBatch> result;
- ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &result));
-}
-
-TEST_F(RecursionLimits, StressLimit) {
- auto CheckDepth = [this](int recursion_depth, bool* it_works) {
- int32_t metadata_length = -1;
- int64_t body_length = -1;
- std::shared_ptr<Schema> schema;
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK(WriteToMmap(
- recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
-
- std::shared_ptr<Message> message;
- ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
- auto metadata = std::make_shared<RecordBatchMetadata>(message);
-
- std::shared_ptr<Buffer> payload;
- ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
-
- io::BufferReader reader(payload);
-
- std::shared_ptr<RecordBatch> result;
- ASSERT_OK(ReadRecordBatch(*metadata, schema, recursion_depth + 1, &reader, &result));
- *it_works = result->Equals(*batch);
- };
-
- bool it_works = false;
- CheckDepth(100, &it_works);
- ASSERT_TRUE(it_works);
-
- CheckDepth(500, &it_works);
- ASSERT_TRUE(it_works);
-}
-
-} // namespace ipc
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/ipc-file-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-file-test.cc b/cpp/src/arrow/ipc/ipc-file-test.cc
deleted file mode 100644
index b457822..0000000
--- a/cpp/src/arrow/ipc/ipc-file-test.cc
+++ /dev/null
@@ -1,228 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstdint>
-#include <cstdio>
-#include <cstring>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "gtest/gtest.h"
-
-#include "arrow/array.h"
-#include "arrow/io/memory.h"
-#include "arrow/io/test-common.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/reader.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/ipc/util.h"
-#include "arrow/ipc/writer.h"
-
-#include "arrow/buffer.h"
-#include "arrow/memory_pool.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-#include "arrow/util/bit-util.h"
-
-namespace arrow {
-namespace ipc {
-
-void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
- if (!left.schema()->Equals(right.schema())) {
- FAIL() << "Left schema: " << left.schema()->ToString()
- << "\nRight schema: " << right.schema()->ToString();
- }
- ASSERT_EQ(left.num_columns(), right.num_columns())
- << left.schema()->ToString() << " result: " << right.schema()->ToString();
- EXPECT_EQ(left.num_rows(), right.num_rows());
- for (int i = 0; i < left.num_columns(); ++i) {
- EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
- << "Idx: " << i << " Name: " << left.column_name(i);
- }
-}
-
-using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
-
-class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
- public:
- void SetUp() {
- pool_ = default_memory_pool();
- buffer_ = std::make_shared<PoolBuffer>(pool_);
- sink_.reset(new io::BufferOutputStream(buffer_));
- }
- void TearDown() {}
-
- Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
- // Write the file
- std::shared_ptr<FileWriter> writer;
- RETURN_NOT_OK(FileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
-
- const int num_batches = static_cast<int>(in_batches.size());
-
- for (const auto& batch : in_batches) {
- RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
- }
- RETURN_NOT_OK(writer->Close());
- RETURN_NOT_OK(sink_->Close());
-
- // Current offset into stream is the end of the file
- int64_t footer_offset;
- RETURN_NOT_OK(sink_->Tell(&footer_offset));
-
- // Open the file
- auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
- std::shared_ptr<FileReader> reader;
- RETURN_NOT_OK(FileReader::Open(buf_reader, footer_offset, &reader));
-
- EXPECT_EQ(num_batches, reader->num_record_batches());
- for (int i = 0; i < num_batches; ++i) {
- std::shared_ptr<RecordBatch> chunk;
- RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
- out_batches->emplace_back(chunk);
- }
-
- return Status::OK();
- }
-
- protected:
- MemoryPool* pool_;
-
- std::unique_ptr<io::BufferOutputStream> sink_;
- std::shared_ptr<PoolBuffer> buffer_;
-};
-
-TEST_P(TestFileFormat, RoundTrip) {
- std::shared_ptr<RecordBatch> batch1;
- std::shared_ptr<RecordBatch> batch2;
- ASSERT_OK((*GetParam())(&batch1)); // NOLINT clang-tidy gtest issue
- ASSERT_OK((*GetParam())(&batch2)); // NOLINT clang-tidy gtest issue
-
- std::vector<std::shared_ptr<RecordBatch>> in_batches = {batch1, batch2};
- std::vector<std::shared_ptr<RecordBatch>> out_batches;
-
- ASSERT_OK(RoundTripHelper(in_batches, &out_batches));
-
- // Compare batches
- for (size_t i = 0; i < in_batches.size(); ++i) {
- CompareBatch(*in_batches[i], *out_batches[i]);
- }
-}
-
-class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
- public:
- void SetUp() {
- pool_ = default_memory_pool();
- buffer_ = std::make_shared<PoolBuffer>(pool_);
- sink_.reset(new io::BufferOutputStream(buffer_));
- }
- void TearDown() {}
-
- Status RoundTripHelper(
- const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
- // Write the file
- std::shared_ptr<StreamWriter> writer;
- RETURN_NOT_OK(StreamWriter::Open(sink_.get(), batch.schema(), &writer));
- int num_batches = 5;
- for (int i = 0; i < num_batches; ++i) {
- RETURN_NOT_OK(writer->WriteRecordBatch(batch));
- }
- RETURN_NOT_OK(writer->Close());
- RETURN_NOT_OK(sink_->Close());
-
- // Open the file
- auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
-
- std::shared_ptr<StreamReader> reader;
- RETURN_NOT_OK(StreamReader::Open(buf_reader, &reader));
-
- std::shared_ptr<RecordBatch> chunk;
- while (true) {
- RETURN_NOT_OK(reader->GetNextRecordBatch(&chunk));
- if (chunk == nullptr) { break; }
- out_batches->emplace_back(chunk);
- }
- return Status::OK();
- }
-
- protected:
- MemoryPool* pool_;
-
- std::unique_ptr<io::BufferOutputStream> sink_;
- std::shared_ptr<PoolBuffer> buffer_;
-};
-
-TEST_P(TestStreamFormat, RoundTrip) {
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
-
- std::vector<std::shared_ptr<RecordBatch>> out_batches;
-
- ASSERT_OK(RoundTripHelper(*batch, &out_batches));
-
- // Compare batches. Same
- for (size_t i = 0; i < out_batches.size(); ++i) {
- CompareBatch(*batch, *out_batches[i]);
- }
-}
-
-#define BATCH_CASES() \
- ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
- &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch, \
- &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate, &MakeTimestamps, &MakeTimes, \
- &MakeFWBinary);
-
-INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
-INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
-
-void CheckBatchDictionaries(const RecordBatch& batch) {
- // Check that dictionaries that should be the same are the same
- auto schema = batch.schema();
-
- const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type);
- const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type);
-
- ASSERT_EQ(t0.dictionary().get(), t1.dictionary().get());
-
- // Same dictionary used for list values
- const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type);
- const auto& t3_value = static_cast<const DictionaryType&>(*t3.value_type());
- ASSERT_EQ(t0.dictionary().get(), t3_value.dictionary().get());
-}
-
-TEST_F(TestStreamFormat, DictionaryRoundTrip) {
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK(MakeDictionary(&batch));
-
- std::vector<std::shared_ptr<RecordBatch>> out_batches;
- ASSERT_OK(RoundTripHelper(*batch, &out_batches));
-
- CheckBatchDictionaries(*out_batches[0]);
-}
-
-TEST_F(TestFileFormat, DictionaryRoundTrip) {
- std::shared_ptr<RecordBatch> batch;
- ASSERT_OK(MakeDictionary(&batch));
-
- std::vector<std::shared_ptr<RecordBatch>> out_batches;
- ASSERT_OK(RoundTripHelper({batch}, &out_batches));
-
- CheckBatchDictionaries(*out_batches[0]);
-}
-
-} // namespace ipc
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/ipc-metadata-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-metadata-test.cc b/cpp/src/arrow/ipc/ipc-metadata-test.cc
deleted file mode 100644
index 4fb3204..0000000
--- a/cpp/src/arrow/ipc/ipc-metadata-test.cc
+++ /dev/null
@@ -1,100 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <memory>
-#include <sstream>
-#include <string>
-
-#include "gtest/gtest.h"
-
-#include "arrow/io/memory.h"
-#include "arrow/ipc/metadata-internal.h"
-#include "arrow/ipc/metadata.h"
-#include "arrow/ipc/test-common.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/test-util.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-class Buffer;
-
-namespace ipc {
-
-class TestSchemaMetadata : public ::testing::Test {
- public:
- void SetUp() {}
-
- void CheckRoundtrip(const Schema& schema, DictionaryMemo* memo) {
- std::shared_ptr<Buffer> buffer;
- ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
-
- std::shared_ptr<Message> message;
- ASSERT_OK(Message::Open(buffer, 0, &message));
-
- ASSERT_EQ(Message::SCHEMA, message->type());
-
- auto schema_msg = std::make_shared<SchemaMetadata>(message);
- ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
-
- DictionaryMemo empty_memo;
-
- std::shared_ptr<Schema> schema2;
- ASSERT_OK(schema_msg->GetSchema(empty_memo, &schema2));
-
- AssertSchemaEqual(schema, *schema2);
- }
-};
-
-const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
-
-TEST_F(TestSchemaMetadata, PrimitiveFields) {
- auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
- auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>(), false);
- auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
- auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
- auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
- auto f5 = std::make_shared<Field>("f5", std::make_shared<UInt16Type>());
- auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
- auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
- auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
- auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>(), false);
- auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
-
- Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
- DictionaryMemo memo;
-
- CheckRoundtrip(schema, &memo);
-}
-
-TEST_F(TestSchemaMetadata, NestedFields) {
- auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
- auto f0 = std::make_shared<Field>("f0", type);
-
- std::shared_ptr<StructType> type2(new StructType({std::make_shared<Field>("k1", INT32),
- std::make_shared<Field>("k2", INT32), std::make_shared<Field>("k3", INT32)}));
- auto f1 = std::make_shared<Field>("f1", type2);
-
- Schema schema({f0, f1});
- DictionaryMemo memo;
-
- CheckRoundtrip(schema, &memo);
-}
-
-} // namespace ipc
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/ipc-read-write-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
new file mode 100644
index 0000000..261ca1d
--- /dev/null
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -0,0 +1,608 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+#include "arrow/array.h"
+#include "arrow/io/memory.h"
+#include "arrow/io/test-common.h"
+#include "arrow/ipc/api.h"
+#include "arrow/ipc/test-common.h"
+#include "arrow/ipc/util.h"
+
+#include "arrow/buffer.h"
+#include "arrow/memory_pool.h"
+#include "arrow/pretty_print.h"
+#include "arrow/status.h"
+#include "arrow/test-util.h"
+#include "arrow/util/bit-util.h"
+
+namespace arrow {
+namespace ipc {
+
+void CompareBatch(const RecordBatch& left, const RecordBatch& right) {
+ if (!left.schema()->Equals(right.schema())) {
+ FAIL() << "Left schema: " << left.schema()->ToString()
+ << "\nRight schema: " << right.schema()->ToString();
+ }
+ ASSERT_EQ(left.num_columns(), right.num_columns())
+ << left.schema()->ToString() << " result: " << right.schema()->ToString();
+ EXPECT_EQ(left.num_rows(), right.num_rows());
+ for (int i = 0; i < left.num_columns(); ++i) {
+ EXPECT_TRUE(left.column(i)->Equals(right.column(i)))
+ << "Idx: " << i << " Name: " << left.column_name(i);
+ }
+}
+
+using BatchVector = std::vector<std::shared_ptr<RecordBatch>>;
+
+class TestSchemaMetadata : public ::testing::Test {
+ public:
+ void SetUp() {}
+
+ void CheckRoundtrip(const Schema& schema, DictionaryMemo* memo) {
+ std::shared_ptr<Buffer> buffer;
+ ASSERT_OK(WriteSchemaMessage(schema, memo, &buffer));
+
+ std::shared_ptr<Message> message;
+ ASSERT_OK(Message::Open(buffer, 0, &message));
+
+ ASSERT_EQ(Message::SCHEMA, message->type());
+
+ auto schema_msg = std::make_shared<SchemaMetadata>(message);
+ ASSERT_EQ(schema.num_fields(), schema_msg->num_fields());
+
+ DictionaryMemo empty_memo;
+
+ std::shared_ptr<Schema> schema2;
+ ASSERT_OK(schema_msg->GetSchema(empty_memo, &schema2));
+
+ AssertSchemaEqual(schema, *schema2);
+ }
+};
+
+const std::shared_ptr<DataType> INT32 = std::make_shared<Int32Type>();
+
+TEST_F(TestSchemaMetadata, PrimitiveFields) {
+ auto f0 = std::make_shared<Field>("f0", std::make_shared<Int8Type>());
+ auto f1 = std::make_shared<Field>("f1", std::make_shared<Int16Type>(), false);
+ auto f2 = std::make_shared<Field>("f2", std::make_shared<Int32Type>());
+ auto f3 = std::make_shared<Field>("f3", std::make_shared<Int64Type>());
+ auto f4 = std::make_shared<Field>("f4", std::make_shared<UInt8Type>());
+ auto f5 = std::make_shared<Field>("f5", std::make_shared<UInt16Type>());
+ auto f6 = std::make_shared<Field>("f6", std::make_shared<UInt32Type>());
+ auto f7 = std::make_shared<Field>("f7", std::make_shared<UInt64Type>());
+ auto f8 = std::make_shared<Field>("f8", std::make_shared<FloatType>());
+ auto f9 = std::make_shared<Field>("f9", std::make_shared<DoubleType>(), false);
+ auto f10 = std::make_shared<Field>("f10", std::make_shared<BooleanType>());
+
+ Schema schema({f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10});
+ DictionaryMemo memo;
+
+ CheckRoundtrip(schema, &memo);
+}
+
+TEST_F(TestSchemaMetadata, NestedFields) {
+ auto type = std::make_shared<ListType>(std::make_shared<Int32Type>());
+ auto f0 = std::make_shared<Field>("f0", type);
+
+ std::shared_ptr<StructType> type2(new StructType({std::make_shared<Field>("k1", INT32),
+ std::make_shared<Field>("k2", INT32), std::make_shared<Field>("k3", INT32)}));
+ auto f1 = std::make_shared<Field>("f1", type2);
+
+ Schema schema({f0, f1});
+ DictionaryMemo memo;
+
+ CheckRoundtrip(schema, &memo);
+}
+
+#define BATCH_CASES() \
+ ::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
+ &MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, &MakeStringTypesRecordBatch, \
+ &MakeStruct, &MakeUnion, &MakeDictionary, &MakeDate, &MakeTimestamps, &MakeTimes, \
+ &MakeFWBinary);
+
+class IpcTestFixture : public io::MemoryMapFixture {
+ public:
+ Status DoStandardRoundTrip(const RecordBatch& batch, bool zero_data,
+ std::shared_ptr<RecordBatch>* batch_result) {
+ int32_t metadata_length;
+ int64_t body_length;
+
+ const int64_t buffer_offset = 0;
+
+ if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
+ RETURN_NOT_OK(mmap_->Seek(0));
+
+ RETURN_NOT_OK(WriteRecordBatch(
+ batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+
+ std::shared_ptr<Message> message;
+ RETURN_NOT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+ auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+ // The buffer offsets start at 0, so we must construct a
+ // RandomAccessFile according to that frame of reference
+ std::shared_ptr<Buffer> buffer_payload;
+ RETURN_NOT_OK(mmap_->ReadAt(metadata_length, body_length, &buffer_payload));
+ io::BufferReader buffer_reader(buffer_payload);
+
+ return ReadRecordBatch(*metadata, batch.schema(), &buffer_reader, batch_result);
+ }
+
+ Status DoLargeRoundTrip(
+ const RecordBatch& batch, bool zero_data, std::shared_ptr<RecordBatch>* result) {
+ int32_t metadata_length;
+ int64_t body_length;
+
+ const int64_t buffer_offset = 0;
+
+ if (zero_data) { RETURN_NOT_OK(ZeroMemoryMap(mmap_.get())); }
+ RETURN_NOT_OK(mmap_->Seek(0));
+
+ RETURN_NOT_OK(WriteLargeRecordBatch(
+ batch, buffer_offset, mmap_.get(), &metadata_length, &body_length, pool_));
+ return ReadLargeRecordBatch(batch.schema(), 0, mmap_.get(), result);
+ }
+
+ void CheckReadResult(const RecordBatch& result, const RecordBatch& expected) {
+ EXPECT_EQ(expected.num_rows(), result.num_rows());
+
+ ASSERT_TRUE(expected.schema()->Equals(result.schema()));
+ ASSERT_EQ(expected.num_columns(), result.num_columns())
+ << expected.schema()->ToString() << " result: " << result.schema()->ToString();
+
+ for (int i = 0; i < expected.num_columns(); ++i) {
+ const auto& left = *expected.column(i);
+ const auto& right = *result.column(i);
+ if (!left.Equals(right)) {
+ std::stringstream pp_result;
+ std::stringstream pp_expected;
+
+ ASSERT_OK(PrettyPrint(left, 0, &pp_expected));
+ ASSERT_OK(PrettyPrint(right, 0, &pp_result));
+
+ FAIL() << "Index: " << i << " Expected: " << pp_expected.str()
+ << "\nGot: " << pp_result.str();
+ }
+ }
+ }
+
+ void CheckRoundtrip(const RecordBatch& batch, int64_t buffer_size) {
+ std::string path = "test-write-row-batch";
+ ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(buffer_size, path, &mmap_));
+
+ std::shared_ptr<RecordBatch> result;
+ ASSERT_OK(DoStandardRoundTrip(batch, true, &result));
+ CheckReadResult(*result, batch);
+
+ ASSERT_OK(DoLargeRoundTrip(batch, true, &result));
+ CheckReadResult(*result, batch);
+ }
+
+ void CheckRoundtrip(const std::shared_ptr<Array>& array, int64_t buffer_size) {
+ auto f0 = arrow::field("f0", array->type());
+ std::vector<std::shared_ptr<Field>> fields = {f0};
+ auto schema = std::make_shared<Schema>(fields);
+
+ RecordBatch batch(schema, 0, {array});
+ CheckRoundtrip(batch, buffer_size);
+ }
+
+ protected:
+ std::shared_ptr<io::MemoryMappedFile> mmap_;
+ MemoryPool* pool_;
+};
+
+class TestWriteRecordBatch : public ::testing::Test, public IpcTestFixture {
+ public:
+ void SetUp() { pool_ = default_memory_pool(); }
+ void TearDown() { io::MemoryMapFixture::TearDown(); }
+};
+
+class TestIpcRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*>,
+ public IpcTestFixture {
+ public:
+ void SetUp() { pool_ = default_memory_pool(); }
+ void TearDown() { io::MemoryMapFixture::TearDown(); }
+};
+
+TEST_P(TestIpcRoundTrip, RoundTrip) {
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
+
+ CheckRoundtrip(*batch, 1 << 20);
+}
+
+TEST_P(TestIpcRoundTrip, SliceRoundTrip) {
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
+
+ // Skip the zero-length case
+ if (batch->num_rows() < 2) { return; }
+
+ auto sliced_batch = batch->Slice(2, 10);
+ CheckRoundtrip(*sliced_batch, 1 << 20);
+}
+
+TEST_P(TestIpcRoundTrip, ZeroLengthArrays) {
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
+
+ std::shared_ptr<RecordBatch> zero_length_batch;
+ if (batch->num_rows() > 2) {
+ zero_length_batch = batch->Slice(2, 0);
+ } else {
+ zero_length_batch = batch->Slice(0, 0);
+ }
+
+ CheckRoundtrip(*zero_length_batch, 1 << 20);
+
+ // ARROW-544: check binary array
+ std::shared_ptr<MutableBuffer> value_offsets;
+ ASSERT_OK(AllocateBuffer(pool_, sizeof(int32_t), &value_offsets));
+ *reinterpret_cast<int32_t*>(value_offsets->mutable_data()) = 0;
+
+ std::shared_ptr<Array> bin_array = std::make_shared<BinaryArray>(0, value_offsets,
+ std::make_shared<Buffer>(nullptr, 0), std::make_shared<Buffer>(nullptr, 0));
+
+ // null value_offsets
+ std::shared_ptr<Array> bin_array2 = std::make_shared<BinaryArray>(0, nullptr, nullptr);
+
+ CheckRoundtrip(bin_array, 1 << 20);
+ CheckRoundtrip(bin_array2, 1 << 20);
+}
+
+void TestGetRecordBatchSize(std::shared_ptr<RecordBatch> batch) {
+ ipc::MockOutputStream mock;
+ int32_t mock_metadata_length = -1;
+ int64_t mock_body_length = -1;
+ int64_t size = -1;
+ ASSERT_OK(WriteRecordBatch(
+ *batch, 0, &mock, &mock_metadata_length, &mock_body_length, default_memory_pool()));
+ ASSERT_OK(GetRecordBatchSize(*batch, &size));
+ ASSERT_EQ(mock.GetExtentBytesWritten(), size);
+}
+
+TEST_F(TestWriteRecordBatch, IntegerGetRecordBatchSize) {
+ std::shared_ptr<RecordBatch> batch;
+
+ ASSERT_OK(MakeIntRecordBatch(&batch));
+ TestGetRecordBatchSize(batch);
+
+ ASSERT_OK(MakeListRecordBatch(&batch));
+ TestGetRecordBatchSize(batch);
+
+ ASSERT_OK(MakeZeroLengthRecordBatch(&batch));
+ TestGetRecordBatchSize(batch);
+
+ ASSERT_OK(MakeNonNullRecordBatch(&batch));
+ TestGetRecordBatchSize(batch);
+
+ ASSERT_OK(MakeDeeplyNestedList(&batch));
+ TestGetRecordBatchSize(batch);
+}
+
+class RecursionLimits : public ::testing::Test, public io::MemoryMapFixture {
+ public:
+ void SetUp() { pool_ = default_memory_pool(); }
+ void TearDown() { io::MemoryMapFixture::TearDown(); }
+
+ Status WriteToMmap(int recursion_level, bool override_level, int32_t* metadata_length,
+ int64_t* body_length, std::shared_ptr<RecordBatch>* batch,
+ std::shared_ptr<Schema>* schema) {
+ const int batch_length = 5;
+ TypePtr type = int32();
+ std::shared_ptr<Array> array;
+ const bool include_nulls = true;
+ RETURN_NOT_OK(MakeRandomInt32Array(1000, include_nulls, pool_, &array));
+ for (int i = 0; i < recursion_level; ++i) {
+ type = list(type);
+ RETURN_NOT_OK(
+ MakeRandomListArray(array, batch_length, include_nulls, pool_, &array));
+ }
+
+ auto f0 = field("f0", type);
+
+ *schema = std::shared_ptr<Schema>(new Schema({f0}));
+
+ std::vector<std::shared_ptr<Array>> arrays = {array};
+ *batch = std::make_shared<RecordBatch>(*schema, batch_length, arrays);
+
+ std::string path = "test-write-past-max-recursion";
+ const int memory_map_size = 1 << 20;
+ io::MemoryMapFixture::InitMemoryMap(memory_map_size, path, &mmap_);
+
+ if (override_level) {
+ return WriteRecordBatch(**batch, 0, mmap_.get(), metadata_length, body_length,
+ pool_, recursion_level + 1);
+ } else {
+ return WriteRecordBatch(
+ **batch, 0, mmap_.get(), metadata_length, body_length, pool_);
+ }
+ }
+
+ protected:
+ std::shared_ptr<io::MemoryMappedFile> mmap_;
+ MemoryPool* pool_;
+};
+
+TEST_F(RecursionLimits, WriteLimit) {
+ int32_t metadata_length = -1;
+ int64_t body_length = -1;
+ std::shared_ptr<Schema> schema;
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_RAISES(Invalid,
+ WriteToMmap((1 << 8) + 1, false, &metadata_length, &body_length, &batch, &schema));
+}
+
+TEST_F(RecursionLimits, ReadLimit) {
+ int32_t metadata_length = -1;
+ int64_t body_length = -1;
+ std::shared_ptr<Schema> schema;
+
+ const int recursion_depth = 64;
+
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK(WriteToMmap(
+ recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+
+ std::shared_ptr<Message> message;
+ ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+ auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+ std::shared_ptr<Buffer> payload;
+ ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
+
+ io::BufferReader reader(payload);
+
+ std::shared_ptr<RecordBatch> result;
+ ASSERT_RAISES(Invalid, ReadRecordBatch(*metadata, schema, &reader, &result));
+}
+
+TEST_F(RecursionLimits, StressLimit) {
+ auto CheckDepth = [this](int recursion_depth, bool* it_works) {
+ int32_t metadata_length = -1;
+ int64_t body_length = -1;
+ std::shared_ptr<Schema> schema;
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK(WriteToMmap(
+ recursion_depth, true, &metadata_length, &body_length, &batch, &schema));
+
+ std::shared_ptr<Message> message;
+ ASSERT_OK(ReadMessage(0, metadata_length, mmap_.get(), &message));
+ auto metadata = std::make_shared<RecordBatchMetadata>(message);
+
+ std::shared_ptr<Buffer> payload;
+ ASSERT_OK(mmap_->ReadAt(metadata_length, body_length, &payload));
+
+ io::BufferReader reader(payload);
+
+ std::shared_ptr<RecordBatch> result;
+ ASSERT_OK(ReadRecordBatch(*metadata, schema, recursion_depth + 1, &reader, &result));
+ *it_works = result->Equals(*batch);
+ };
+
+ bool it_works = false;
+ CheckDepth(100, &it_works);
+ ASSERT_TRUE(it_works);
+
+ CheckDepth(500, &it_works);
+ ASSERT_TRUE(it_works);
+}
+
+class TestFileFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+ void SetUp() {
+ pool_ = default_memory_pool();
+ buffer_ = std::make_shared<PoolBuffer>(pool_);
+ sink_.reset(new io::BufferOutputStream(buffer_));
+ }
+ void TearDown() {}
+
+ Status RoundTripHelper(const BatchVector& in_batches, BatchVector* out_batches) {
+ // Write the file
+ std::shared_ptr<FileWriter> writer;
+ RETURN_NOT_OK(FileWriter::Open(sink_.get(), in_batches[0]->schema(), &writer));
+
+ const int num_batches = static_cast<int>(in_batches.size());
+
+ for (const auto& batch : in_batches) {
+ RETURN_NOT_OK(writer->WriteRecordBatch(*batch));
+ }
+ RETURN_NOT_OK(writer->Close());
+ RETURN_NOT_OK(sink_->Close());
+
+ // Current offset into stream is the end of the file
+ int64_t footer_offset;
+ RETURN_NOT_OK(sink_->Tell(&footer_offset));
+
+ // Open the file
+ auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+ std::shared_ptr<FileReader> reader;
+ RETURN_NOT_OK(FileReader::Open(buf_reader, footer_offset, &reader));
+
+ EXPECT_EQ(num_batches, reader->num_record_batches());
+ for (int i = 0; i < num_batches; ++i) {
+ std::shared_ptr<RecordBatch> chunk;
+ RETURN_NOT_OK(reader->GetRecordBatch(i, &chunk));
+ out_batches->emplace_back(chunk);
+ }
+
+ return Status::OK();
+ }
+
+ protected:
+ MemoryPool* pool_;
+
+ std::unique_ptr<io::BufferOutputStream> sink_;
+ std::shared_ptr<PoolBuffer> buffer_;
+};
+
+TEST_P(TestFileFormat, RoundTrip) {
+ std::shared_ptr<RecordBatch> batch1;
+ std::shared_ptr<RecordBatch> batch2;
+ ASSERT_OK((*GetParam())(&batch1)); // NOLINT clang-tidy gtest issue
+ ASSERT_OK((*GetParam())(&batch2)); // NOLINT clang-tidy gtest issue
+
+ std::vector<std::shared_ptr<RecordBatch>> in_batches = {batch1, batch2};
+ std::vector<std::shared_ptr<RecordBatch>> out_batches;
+
+ ASSERT_OK(RoundTripHelper(in_batches, &out_batches));
+
+ // Compare batches
+ for (size_t i = 0; i < in_batches.size(); ++i) {
+ CompareBatch(*in_batches[i], *out_batches[i]);
+ }
+}
+
+class TestStreamFormat : public ::testing::TestWithParam<MakeRecordBatch*> {
+ public:
+ void SetUp() {
+ pool_ = default_memory_pool();
+ buffer_ = std::make_shared<PoolBuffer>(pool_);
+ sink_.reset(new io::BufferOutputStream(buffer_));
+ }
+ void TearDown() {}
+
+ Status RoundTripHelper(
+ const RecordBatch& batch, std::vector<std::shared_ptr<RecordBatch>>* out_batches) {
+ // Write the file
+ std::shared_ptr<StreamWriter> writer;
+ RETURN_NOT_OK(StreamWriter::Open(sink_.get(), batch.schema(), &writer));
+ int num_batches = 5;
+ for (int i = 0; i < num_batches; ++i) {
+ RETURN_NOT_OK(writer->WriteRecordBatch(batch));
+ }
+ RETURN_NOT_OK(writer->Close());
+ RETURN_NOT_OK(sink_->Close());
+
+ // Open the file
+ auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+
+ std::shared_ptr<StreamReader> reader;
+ RETURN_NOT_OK(StreamReader::Open(buf_reader, &reader));
+
+ std::shared_ptr<RecordBatch> chunk;
+ while (true) {
+ RETURN_NOT_OK(reader->GetNextRecordBatch(&chunk));
+ if (chunk == nullptr) { break; }
+ out_batches->emplace_back(chunk);
+ }
+ return Status::OK();
+ }
+
+ protected:
+ MemoryPool* pool_;
+
+ std::unique_ptr<io::BufferOutputStream> sink_;
+ std::shared_ptr<PoolBuffer> buffer_;
+};
+
+TEST_P(TestStreamFormat, RoundTrip) {
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK((*GetParam())(&batch)); // NOLINT clang-tidy gtest issue
+
+ std::vector<std::shared_ptr<RecordBatch>> out_batches;
+
+ ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+
+ // Compare batches. Same
+ for (size_t i = 0; i < out_batches.size(); ++i) {
+ CompareBatch(*batch, *out_batches[i]);
+ }
+}
+
+INSTANTIATE_TEST_CASE_P(GenericIpcRoundTripTests, TestIpcRoundTrip, BATCH_CASES());
+INSTANTIATE_TEST_CASE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
+INSTANTIATE_TEST_CASE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
+
+TEST_F(TestIpcRoundTrip, LargeRecordBatch) {
+ const int64_t length = static_cast<int64_t>(std::numeric_limits<int32_t>::max()) + 1;
+
+ BooleanBuilder builder(default_memory_pool());
+ ASSERT_OK(builder.Reserve(length));
+ ASSERT_OK(builder.Advance(length));
+
+ std::shared_ptr<Array> array;
+ ASSERT_OK(builder.Finish(&array));
+
+ auto f0 = arrow::field("f0", array->type());
+ std::vector<std::shared_ptr<Field>> fields = {f0};
+ auto schema = std::make_shared<Schema>(fields);
+
+ RecordBatch batch(schema, 0, {array});
+
+ std::string path = "test-write-large-record_batch";
+
+ // 512 MB
+ constexpr int64_t kBufferSize = 1 << 29;
+
+ ASSERT_OK(io::MemoryMapFixture::InitMemoryMap(kBufferSize, path, &mmap_));
+
+ std::shared_ptr<RecordBatch> result;
+ ASSERT_OK(DoLargeRoundTrip(batch, false, &result));
+ CheckReadResult(*result, batch);
+
+ // Fails if we try to write this with the normal code path
+ ASSERT_RAISES(Invalid, DoStandardRoundTrip(batch, false, &result));
+}
+
+void CheckBatchDictionaries(const RecordBatch& batch) {
+ // Check that dictionaries that should be the same are the same
+ auto schema = batch.schema();
+
+ const auto& t0 = static_cast<const DictionaryType&>(*schema->field(0)->type);
+ const auto& t1 = static_cast<const DictionaryType&>(*schema->field(1)->type);
+
+ ASSERT_EQ(t0.dictionary().get(), t1.dictionary().get());
+
+ // Same dictionary used for list values
+ const auto& t3 = static_cast<const ListType&>(*schema->field(3)->type);
+ const auto& t3_value = static_cast<const DictionaryType&>(*t3.value_type());
+ ASSERT_EQ(t0.dictionary().get(), t3_value.dictionary().get());
+}
+
+TEST_F(TestStreamFormat, DictionaryRoundTrip) {
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK(MakeDictionary(&batch));
+
+ std::vector<std::shared_ptr<RecordBatch>> out_batches;
+ ASSERT_OK(RoundTripHelper(*batch, &out_batches));
+
+ CheckBatchDictionaries(*out_batches[0]);
+}
+
+TEST_F(TestFileFormat, DictionaryRoundTrip) {
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK(MakeDictionary(&batch));
+
+ std::vector<std::shared_ptr<RecordBatch>> out_batches;
+ ASSERT_OK(RoundTripHelper({batch}, &out_batches));
+
+ CheckBatchDictionaries(*out_batches[0]);
+}
+
+} // namespace ipc
+} // namespace arrow
[2/3] arrow git commit: ARROW-661: [C++] Add LargeRecordBatch
metadata type, IPC support, associated refactoring
Posted by uw...@apache.org.
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/metadata-internal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
deleted file mode 100644
index be0d282..0000000
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ /dev/null
@@ -1,597 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "arrow/ipc/metadata-internal.h"
-
-#include <cstdint>
-#include <cstring>
-#include <memory>
-#include <sstream>
-#include <string>
-
-#include "flatbuffers/flatbuffers.h"
-
-#include "arrow/array.h"
-#include "arrow/buffer.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/schema.h"
-#include "arrow/status.h"
-#include "arrow/type.h"
-
-namespace arrow {
-
-namespace flatbuf = org::apache::arrow::flatbuf;
-
-namespace ipc {
-
-static Status IntFromFlatbuffer(
- const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
- if (int_data->bitWidth() > 64) {
- return Status::NotImplemented("Integers with more than 64 bits not implemented");
- }
- if (int_data->bitWidth() < 8) {
- return Status::NotImplemented("Integers with less than 8 bits not implemented");
- }
-
- switch (int_data->bitWidth()) {
- case 8:
- *out = int_data->is_signed() ? int8() : uint8();
- break;
- case 16:
- *out = int_data->is_signed() ? int16() : uint16();
- break;
- case 32:
- *out = int_data->is_signed() ? int32() : uint32();
- break;
- case 64:
- *out = int_data->is_signed() ? int64() : uint64();
- break;
- default:
- return Status::NotImplemented("Integers not in cstdint are not implemented");
- }
- return Status::OK();
-}
-
-static Status FloatFromFlatuffer(
- const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
- if (float_data->precision() == flatbuf::Precision_HALF) {
- *out = float16();
- } else if (float_data->precision() == flatbuf::Precision_SINGLE) {
- *out = float32();
- } else {
- *out = float64();
- }
- return Status::OK();
-}
-
-// Forward declaration
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
- DictionaryMemo* dictionary_memo, FieldOffset* offset);
-
-static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
- return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
-}
-
-static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
- return flatbuf::CreateFloatingPoint(fbb, precision).Union();
-}
-
-static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
- std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo) {
- FieldOffset field;
- for (int i = 0; i < type->num_children(); ++i) {
- RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
- out_children->push_back(field);
- }
- return Status::OK();
-}
-
-static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
- std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
- Offset* offset) {
- RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
- *offset = flatbuf::CreateList(fbb).Union();
- return Status::OK();
-}
-
-static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
- std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
- Offset* offset) {
- RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
- *offset = flatbuf::CreateStruct_(fbb).Union();
- return Status::OK();
-}
-
-// ----------------------------------------------------------------------
-// Union implementation
-
-static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
- const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
- UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
- : UnionMode::DENSE;
-
- std::vector<uint8_t> type_codes;
-
- const flatbuffers::Vector<int32_t>* fb_type_ids = union_data->typeIds();
- if (fb_type_ids == nullptr) {
- for (uint8_t i = 0; i < children.size(); ++i) {
- type_codes.push_back(i);
- }
- } else {
- for (int32_t id : (*fb_type_ids)) {
- // TODO(wesm): can these values exceed 255?
- type_codes.push_back(static_cast<uint8_t>(id));
- }
- }
-
- *out = union_(children, type_codes, mode);
- return Status::OK();
-}
-
-static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
- std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
- Offset* offset) {
- RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
-
- const auto& union_type = static_cast<const UnionType&>(*type);
-
- flatbuf::UnionMode mode = union_type.mode == UnionMode::SPARSE
- ? flatbuf::UnionMode_Sparse
- : flatbuf::UnionMode_Dense;
-
- std::vector<int32_t> type_ids;
- type_ids.reserve(union_type.type_codes.size());
- for (uint8_t code : union_type.type_codes) {
- type_ids.push_back(code);
- }
-
- auto fb_type_ids = fbb.CreateVector(type_ids);
-
- *offset = flatbuf::CreateUnion(fbb, mode, fb_type_ids).Union();
- return Status::OK();
-}
-
-#define INT_TO_FB_CASE(BIT_WIDTH, IS_SIGNED) \
- *out_type = flatbuf::Type_Int; \
- *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
- break;
-
-static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
- switch (unit) {
- case TimeUnit::SECOND:
- return flatbuf::TimeUnit_SECOND;
- case TimeUnit::MILLI:
- return flatbuf::TimeUnit_MILLISECOND;
- case TimeUnit::MICRO:
- return flatbuf::TimeUnit_MICROSECOND;
- case TimeUnit::NANO:
- return flatbuf::TimeUnit_NANOSECOND;
- default:
- break;
- }
- return flatbuf::TimeUnit_MIN;
-}
-
-static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) {
- switch (unit) {
- case flatbuf::TimeUnit_SECOND:
- return TimeUnit::SECOND;
- case flatbuf::TimeUnit_MILLISECOND:
- return TimeUnit::MILLI;
- case flatbuf::TimeUnit_MICROSECOND:
- return TimeUnit::MICRO;
- case flatbuf::TimeUnit_NANOSECOND:
- return TimeUnit::NANO;
- default:
- break;
- }
- // cannot reach
- return TimeUnit::SECOND;
-}
-
-static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
- const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
- switch (type) {
- case flatbuf::Type_NONE:
- return Status::Invalid("Type metadata cannot be none");
- case flatbuf::Type_Int:
- return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
- case flatbuf::Type_FloatingPoint:
- return FloatFromFlatuffer(
- static_cast<const flatbuf::FloatingPoint*>(type_data), out);
- case flatbuf::Type_Binary:
- *out = binary();
- return Status::OK();
- case flatbuf::Type_FixedWidthBinary: {
- auto fw_binary = static_cast<const flatbuf::FixedWidthBinary*>(type_data);
- *out = fixed_width_binary(fw_binary->byteWidth());
- return Status::OK();
- }
- case flatbuf::Type_Utf8:
- *out = utf8();
- return Status::OK();
- case flatbuf::Type_Bool:
- *out = boolean();
- return Status::OK();
- case flatbuf::Type_Decimal:
- return Status::NotImplemented("Decimal");
- case flatbuf::Type_Date:
- *out = date();
- return Status::OK();
- case flatbuf::Type_Time: {
- auto time_type = static_cast<const flatbuf::Time*>(type_data);
- *out = time(FromFlatbufferUnit(time_type->unit()));
- return Status::OK();
- }
- case flatbuf::Type_Timestamp: {
- auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
- *out = timestamp(FromFlatbufferUnit(ts_type->unit()));
- return Status::OK();
- }
- case flatbuf::Type_Interval:
- return Status::NotImplemented("Interval");
- case flatbuf::Type_List:
- if (children.size() != 1) {
- return Status::Invalid("List must have exactly 1 child field");
- }
- *out = std::make_shared<ListType>(children[0]);
- return Status::OK();
- case flatbuf::Type_Struct_:
- *out = std::make_shared<StructType>(children);
- return Status::OK();
- case flatbuf::Type_Union:
- return UnionFromFlatbuffer(
- static_cast<const flatbuf::Union*>(type_data), children, out);
- default:
- return Status::Invalid("Unrecognized type");
- }
-}
-
-// TODO(wesm): Convert this to visitor pattern
-static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
- std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
- flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
- if (type->type == Type::DICTIONARY) {
- // In this library, the dictionary "type" is a logical construct. Here we
- // pass through to the value type, as we've already captured the index
- // type in the DictionaryEncoding metadata in the parent field
- const auto& dict_type = static_cast<const DictionaryType&>(*type);
- return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
- out_type, dictionary_memo, offset);
- }
-
- std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
- for (const BufferDescr& descr : buffer_layout) {
- flatbuf::VectorType vector_type;
- switch (descr.type()) {
- case BufferType::OFFSET:
- vector_type = flatbuf::VectorType_OFFSET;
- break;
- case BufferType::DATA:
- vector_type = flatbuf::VectorType_DATA;
- break;
- case BufferType::VALIDITY:
- vector_type = flatbuf::VectorType_VALIDITY;
- break;
- case BufferType::TYPE:
- vector_type = flatbuf::VectorType_TYPE;
- break;
- default:
- vector_type = flatbuf::VectorType_DATA;
- break;
- }
- auto offset = flatbuf::CreateVectorLayout(
- fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
- layout->push_back(offset);
- }
-
- switch (type->type) {
- case Type::BOOL:
- *out_type = flatbuf::Type_Bool;
- *offset = flatbuf::CreateBool(fbb).Union();
- break;
- case Type::UINT8:
- INT_TO_FB_CASE(8, false);
- case Type::INT8:
- INT_TO_FB_CASE(8, true);
- case Type::UINT16:
- INT_TO_FB_CASE(16, false);
- case Type::INT16:
- INT_TO_FB_CASE(16, true);
- case Type::UINT32:
- INT_TO_FB_CASE(32, false);
- case Type::INT32:
- INT_TO_FB_CASE(32, true);
- case Type::UINT64:
- INT_TO_FB_CASE(64, false);
- case Type::INT64:
- INT_TO_FB_CASE(64, true);
- case Type::FLOAT:
- *out_type = flatbuf::Type_FloatingPoint;
- *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
- break;
- case Type::DOUBLE:
- *out_type = flatbuf::Type_FloatingPoint;
- *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
- break;
- case Type::FIXED_WIDTH_BINARY: {
- const auto& fw_type = static_cast<const FixedWidthBinaryType&>(*type);
- *out_type = flatbuf::Type_FixedWidthBinary;
- *offset = flatbuf::CreateFixedWidthBinary(fbb, fw_type.byte_width()).Union();
- } break;
- case Type::BINARY:
- *out_type = flatbuf::Type_Binary;
- *offset = flatbuf::CreateBinary(fbb).Union();
- break;
- case Type::STRING:
- *out_type = flatbuf::Type_Utf8;
- *offset = flatbuf::CreateUtf8(fbb).Union();
- break;
- case Type::DATE:
- *out_type = flatbuf::Type_Date;
- *offset = flatbuf::CreateDate(fbb).Union();
- break;
- case Type::TIME: {
- const auto& time_type = static_cast<const TimeType&>(*type);
- *out_type = flatbuf::Type_Time;
- *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
- } break;
- case Type::TIMESTAMP: {
- const auto& ts_type = static_cast<const TimestampType&>(*type);
- *out_type = flatbuf::Type_Timestamp;
- *offset = flatbuf::CreateTimestamp(fbb, ToFlatbufferUnit(ts_type.unit)).Union();
- } break;
- case Type::LIST:
- *out_type = flatbuf::Type_List;
- return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
- case Type::STRUCT:
- *out_type = flatbuf::Type_Struct_;
- return StructToFlatbuffer(fbb, type, children, dictionary_memo, offset);
- case Type::UNION:
- *out_type = flatbuf::Type_Union;
- return UnionToFlatBuffer(fbb, type, children, dictionary_memo, offset);
- default:
- *out_type = flatbuf::Type_NONE; // Make clang-tidy happy
- std::stringstream ss;
- ss << "Unable to convert type: " << type->ToString() << std::endl;
- return Status::NotImplemented(ss.str());
- }
- return Status::OK();
-}
-
-using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
-
-static DictionaryOffset GetDictionaryEncoding(
- FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
- int64_t dictionary_id = memo->GetId(type.dictionary());
-
- // We assume that the dictionary index type (as an integer) has already been
- // validated elsewhere, and can safely assume we are dealing with signed
- // integers
- const auto& fw_index_type = static_cast<const FixedWidthType&>(*type.index_type());
-
- auto index_type_offset = flatbuf::CreateInt(fbb, fw_index_type.bit_width(), true);
-
- // TODO(wesm): ordered dictionaries
- return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset);
-}
-
-static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
- DictionaryMemo* dictionary_memo, FieldOffset* offset) {
- auto fb_name = fbb.CreateString(field->name);
-
- flatbuf::Type type_enum;
- Offset type_offset;
- Offset type_layout;
- std::vector<FieldOffset> children;
- std::vector<VectorLayoutOffset> layout;
-
- RETURN_NOT_OK(TypeToFlatbuffer(
- fbb, field->type, &children, &layout, &type_enum, dictionary_memo, &type_offset));
- auto fb_children = fbb.CreateVector(children);
- auto fb_layout = fbb.CreateVector(layout);
-
- DictionaryOffset dictionary = 0;
- if (field->type->type == Type::DICTIONARY) {
- dictionary = GetDictionaryEncoding(
- fbb, static_cast<const DictionaryType&>(*field->type), dictionary_memo);
- }
-
- // TODO: produce the list of VectorTypes
- *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_offset,
- dictionary, fb_children, fb_layout);
-
- return Status::OK();
-}
-
-Status FieldFromFlatbufferDictionary(
- const flatbuf::Field* field, std::shared_ptr<Field>* out) {
- // Need an empty memo to pass down for constructing children
- DictionaryMemo dummy_memo;
-
- // Any DictionaryEncoding set is ignored here
-
- std::shared_ptr<DataType> type;
- auto children = field->children();
- std::vector<std::shared_ptr<Field>> child_fields(children->size());
- for (int i = 0; i < static_cast<int>(children->size()); ++i) {
- RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), dummy_memo, &child_fields[i]));
- }
-
- RETURN_NOT_OK(
- TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
-
- *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
- return Status::OK();
-}
-
-Status FieldFromFlatbuffer(const flatbuf::Field* field,
- const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out) {
- std::shared_ptr<DataType> type;
-
- const flatbuf::DictionaryEncoding* encoding = field->dictionary();
-
- if (encoding == nullptr) {
- // The field is not dictionary encoded. We must potentially visit its
- // children to fully reconstruct the data type
- auto children = field->children();
- std::vector<std::shared_ptr<Field>> child_fields(children->size());
- for (int i = 0; i < static_cast<int>(children->size()); ++i) {
- RETURN_NOT_OK(
- FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
- }
- RETURN_NOT_OK(
- TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
- } else {
- // The field is dictionary encoded. The type of the dictionary values has
- // been determined elsewhere, and is stored in the DictionaryMemo. Here we
- // construct the logical DictionaryType object
-
- std::shared_ptr<Array> dictionary;
- RETURN_NOT_OK(dictionary_memo.GetDictionary(encoding->id(), &dictionary));
-
- std::shared_ptr<DataType> index_type;
- RETURN_NOT_OK(IntFromFlatbuffer(encoding->indexType(), &index_type));
- type = std::make_shared<DictionaryType>(index_type, dictionary);
- }
- *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
- return Status::OK();
-}
-
-// Implement MessageBuilder
-
-// will return the endianness of the system we are running on
-// based the NUMPY_API function. See NOTICE.txt
-flatbuf::Endianness endianness() {
- union {
- uint32_t i;
- char c[4];
- } bint = {0x01020304};
-
- return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
-}
-
-Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
- flatbuffers::Offset<flatbuf::Schema>* out) {
- std::vector<FieldOffset> field_offsets;
- for (int i = 0; i < schema.num_fields(); ++i) {
- std::shared_ptr<Field> field = schema.field(i);
- FieldOffset offset;
- RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, dictionary_memo, &offset));
- field_offsets.push_back(offset);
- }
-
- *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets));
- return Status::OK();
-}
-
-class MessageBuilder {
- public:
- Status SetSchema(const Schema& schema, DictionaryMemo* dictionary_memo) {
- flatbuffers::Offset<flatbuf::Schema> fb_schema;
- RETURN_NOT_OK(SchemaToFlatbuffer(fbb_, schema, dictionary_memo, &fb_schema));
-
- header_type_ = flatbuf::MessageHeader_Schema;
- header_ = fb_schema.Union();
- body_length_ = 0;
- return Status::OK();
- }
-
- Status SetRecordBatch(int32_t length, int64_t body_length,
- const std::vector<flatbuf::FieldNode>& nodes,
- const std::vector<flatbuf::Buffer>& buffers) {
- header_type_ = flatbuf::MessageHeader_RecordBatch;
- header_ = flatbuf::CreateRecordBatch(fbb_, length, fbb_.CreateVectorOfStructs(nodes),
- fbb_.CreateVectorOfStructs(buffers))
- .Union();
- body_length_ = body_length;
-
- return Status::OK();
- }
-
- Status SetDictionary(int64_t id, int32_t length, int64_t body_length,
- const std::vector<flatbuf::FieldNode>& nodes,
- const std::vector<flatbuf::Buffer>& buffers) {
- header_type_ = flatbuf::MessageHeader_DictionaryBatch;
-
- auto record_batch = flatbuf::CreateRecordBatch(fbb_, length,
- fbb_.CreateVectorOfStructs(nodes), fbb_.CreateVectorOfStructs(buffers));
-
- header_ = flatbuf::CreateDictionaryBatch(fbb_, id, record_batch).Union();
- body_length_ = body_length;
- return Status::OK();
- }
-
- Status Finish();
-
- Status GetBuffer(std::shared_ptr<Buffer>* out);
-
- private:
- flatbuf::MessageHeader header_type_;
- flatbuffers::Offset<void> header_;
- int64_t body_length_;
- flatbuffers::FlatBufferBuilder fbb_;
-};
-
-Status WriteSchemaMessage(
- const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out) {
- MessageBuilder message;
- RETURN_NOT_OK(message.SetSchema(schema, dictionary_memo));
- RETURN_NOT_OK(message.Finish());
- return message.GetBuffer(out);
-}
-
-Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
- const std::vector<flatbuf::FieldNode>& nodes,
- const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
- MessageBuilder builder;
- RETURN_NOT_OK(builder.SetRecordBatch(length, body_length, nodes, buffers));
- RETURN_NOT_OK(builder.Finish());
- return builder.GetBuffer(out);
-}
-
-Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
- const std::vector<flatbuf::FieldNode>& nodes,
- const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out) {
- MessageBuilder builder;
- RETURN_NOT_OK(builder.SetDictionary(id, length, body_length, nodes, buffers));
- RETURN_NOT_OK(builder.Finish());
- return builder.GetBuffer(out);
-}
-
-Status MessageBuilder::Finish() {
- auto message =
- flatbuf::CreateMessage(fbb_, kMetadataVersion, header_type_, header_, body_length_);
- fbb_.Finish(message);
- return Status::OK();
-}
-
-Status MessageBuilder::GetBuffer(std::shared_ptr<Buffer>* out) {
- int32_t size = fbb_.GetSize();
-
- auto result = std::make_shared<PoolBuffer>();
- RETURN_NOT_OK(result->Resize(size));
-
- uint8_t* dst = result->mutable_data();
- memcpy(dst, fbb_.GetBufferPointer(), size);
-
- *out = result;
- return Status::OK();
-}
-
-} // namespace ipc
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/metadata-internal.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata-internal.h b/cpp/src/arrow/ipc/metadata-internal.h
deleted file mode 100644
index 59afecb..0000000
--- a/cpp/src/arrow/ipc/metadata-internal.h
+++ /dev/null
@@ -1,83 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_IPC_METADATA_INTERNAL_H
-#define ARROW_IPC_METADATA_INTERNAL_H
-
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include "flatbuffers/flatbuffers.h"
-
-#include "arrow/ipc/File_generated.h"
-#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata.h"
-
-namespace arrow {
-
-namespace flatbuf = org::apache::arrow::flatbuf;
-
-class Buffer;
-struct Field;
-class Schema;
-class Status;
-
-namespace ipc {
-
-using FBB = flatbuffers::FlatBufferBuilder;
-using FieldOffset = flatbuffers::Offset<arrow::flatbuf::Field>;
-using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
-using Offset = flatbuffers::Offset<void>;
-
-static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
-
-// Construct a field with type for a dictionary-encoded field. None of its
-// children or children's descendents can be dictionary encoded
-Status FieldFromFlatbufferDictionary(
- const flatbuf::Field* field, std::shared_ptr<Field>* out);
-
-// Construct a field for a non-dictionary-encoded field. Its children may be
-// dictionary encoded
-Status FieldFromFlatbuffer(const flatbuf::Field* field,
- const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out);
-
-Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema, DictionaryMemo* dictionary_memo,
- flatbuffers::Offset<flatbuf::Schema>* out);
-
-// Serialize arrow::Schema as a Flatbuffer
-//
-// \param[in] schema a Schema instance
-// \param[inout] dictionary_memo class for tracking dictionaries and assigning
-// dictionary ids
-// \param[out] out the serialized arrow::Buffer
-// \return Status outcome
-Status WriteSchemaMessage(
- const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
-
-Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
- const std::vector<flatbuf::FieldNode>& nodes,
- const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
-
-Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
- const std::vector<flatbuf::FieldNode>& nodes,
- const std::vector<flatbuf::Buffer>& buffers, std::shared_ptr<Buffer>* out);
-
-} // namespace ipc
-} // namespace arrow
-
-#endif // ARROW_IPC_METADATA_INTERNAL_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/metadata.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata.cc b/cpp/src/arrow/ipc/metadata.cc
index 71bc5c9..a418d48 100644
--- a/cpp/src/arrow/ipc/metadata.cc
+++ b/cpp/src/arrow/ipc/metadata.cc
@@ -24,14 +24,14 @@
#include "flatbuffers/flatbuffers.h"
+#include "arrow/array.h"
+#include "arrow/buffer.h"
#include "arrow/io/interfaces.h"
#include "arrow/ipc/File_generated.h"
#include "arrow/ipc/Message_generated.h"
-#include "arrow/ipc/metadata-internal.h"
-
-#include "arrow/buffer.h"
#include "arrow/schema.h"
#include "arrow/status.h"
+#include "arrow/type.h"
namespace arrow {
@@ -39,6 +39,643 @@ namespace flatbuf = org::apache::arrow::flatbuf;
namespace ipc {
+using FBB = flatbuffers::FlatBufferBuilder;
+using DictionaryOffset = flatbuffers::Offset<flatbuf::DictionaryEncoding>;
+using FieldOffset = flatbuffers::Offset<flatbuf::Field>;
+using LargeRecordBatchOffset = flatbuffers::Offset<flatbuf::LargeRecordBatch>;
+using RecordBatchOffset = flatbuffers::Offset<flatbuf::RecordBatch>;
+using VectorLayoutOffset = flatbuffers::Offset<arrow::flatbuf::VectorLayout>;
+using Offset = flatbuffers::Offset<void>;
+
+static constexpr flatbuf::MetadataVersion kMetadataVersion = flatbuf::MetadataVersion_V2;
+
+static Status IntFromFlatbuffer(
+ const flatbuf::Int* int_data, std::shared_ptr<DataType>* out) {
+ if (int_data->bitWidth() > 64) {
+ return Status::NotImplemented("Integers with more than 64 bits not implemented");
+ }
+ if (int_data->bitWidth() < 8) {
+ return Status::NotImplemented("Integers with less than 8 bits not implemented");
+ }
+
+ switch (int_data->bitWidth()) {
+ case 8:
+ *out = int_data->is_signed() ? int8() : uint8();
+ break;
+ case 16:
+ *out = int_data->is_signed() ? int16() : uint16();
+ break;
+ case 32:
+ *out = int_data->is_signed() ? int32() : uint32();
+ break;
+ case 64:
+ *out = int_data->is_signed() ? int64() : uint64();
+ break;
+ default:
+ return Status::NotImplemented("Integers not in cstdint are not implemented");
+ }
+ return Status::OK();
+}
+
+static Status FloatFromFlatuffer(
+ const flatbuf::FloatingPoint* float_data, std::shared_ptr<DataType>* out) {
+ if (float_data->precision() == flatbuf::Precision_HALF) {
+ *out = float16();
+ } else if (float_data->precision() == flatbuf::Precision_SINGLE) {
+ *out = float32();
+ } else {
+ *out = float64();
+ }
+ return Status::OK();
+}
+
+// Forward declaration
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+ DictionaryMemo* dictionary_memo, FieldOffset* offset);
+
+static Offset IntToFlatbuffer(FBB& fbb, int bitWidth, bool is_signed) {
+ return flatbuf::CreateInt(fbb, bitWidth, is_signed).Union();
+}
+
+static Offset FloatToFlatbuffer(FBB& fbb, flatbuf::Precision precision) {
+ return flatbuf::CreateFloatingPoint(fbb, precision).Union();
+}
+
+static Status AppendChildFields(FBB& fbb, const std::shared_ptr<DataType>& type,
+ std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo) {
+ FieldOffset field;
+ for (int i = 0; i < type->num_children(); ++i) {
+ RETURN_NOT_OK(FieldToFlatbuffer(fbb, type->child(i), dictionary_memo, &field));
+ out_children->push_back(field);
+ }
+ return Status::OK();
+}
+
+static Status ListToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+ std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+ Offset* offset) {
+ RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+ *offset = flatbuf::CreateList(fbb).Union();
+ return Status::OK();
+}
+
+static Status StructToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+ std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+ Offset* offset) {
+ RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+ *offset = flatbuf::CreateStruct_(fbb).Union();
+ return Status::OK();
+}
+
+// ----------------------------------------------------------------------
+// Union implementation
+
+static Status UnionFromFlatbuffer(const flatbuf::Union* union_data,
+ const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+ UnionMode mode = union_data->mode() == flatbuf::UnionMode_Sparse ? UnionMode::SPARSE
+ : UnionMode::DENSE;
+
+ std::vector<uint8_t> type_codes;
+
+ const flatbuffers::Vector<int32_t>* fb_type_ids = union_data->typeIds();
+ if (fb_type_ids == nullptr) {
+ for (uint8_t i = 0; i < children.size(); ++i) {
+ type_codes.push_back(i);
+ }
+ } else {
+ for (int32_t id : (*fb_type_ids)) {
+ // TODO(wesm): can these values exceed 255?
+ type_codes.push_back(static_cast<uint8_t>(id));
+ }
+ }
+
+ *out = union_(children, type_codes, mode);
+ return Status::OK();
+}
+
+static Status UnionToFlatBuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+ std::vector<FieldOffset>* out_children, DictionaryMemo* dictionary_memo,
+ Offset* offset) {
+ RETURN_NOT_OK(AppendChildFields(fbb, type, out_children, dictionary_memo));
+
+ const auto& union_type = static_cast<const UnionType&>(*type);
+
+ flatbuf::UnionMode mode = union_type.mode == UnionMode::SPARSE
+ ? flatbuf::UnionMode_Sparse
+ : flatbuf::UnionMode_Dense;
+
+ std::vector<int32_t> type_ids;
+ type_ids.reserve(union_type.type_codes.size());
+ for (uint8_t code : union_type.type_codes) {
+ type_ids.push_back(code);
+ }
+
+ auto fb_type_ids = fbb.CreateVector(type_ids);
+
+ *offset = flatbuf::CreateUnion(fbb, mode, fb_type_ids).Union();
+ return Status::OK();
+}
+
+#define INT_TO_FB_CASE(BIT_WIDTH, IS_SIGNED) \
+ *out_type = flatbuf::Type_Int; \
+ *offset = IntToFlatbuffer(fbb, BIT_WIDTH, IS_SIGNED); \
+ break;
+
+static inline flatbuf::TimeUnit ToFlatbufferUnit(TimeUnit unit) {
+ switch (unit) {
+ case TimeUnit::SECOND:
+ return flatbuf::TimeUnit_SECOND;
+ case TimeUnit::MILLI:
+ return flatbuf::TimeUnit_MILLISECOND;
+ case TimeUnit::MICRO:
+ return flatbuf::TimeUnit_MICROSECOND;
+ case TimeUnit::NANO:
+ return flatbuf::TimeUnit_NANOSECOND;
+ default:
+ break;
+ }
+ return flatbuf::TimeUnit_MIN;
+}
+
+static inline TimeUnit FromFlatbufferUnit(flatbuf::TimeUnit unit) {
+ switch (unit) {
+ case flatbuf::TimeUnit_SECOND:
+ return TimeUnit::SECOND;
+ case flatbuf::TimeUnit_MILLISECOND:
+ return TimeUnit::MILLI;
+ case flatbuf::TimeUnit_MICROSECOND:
+ return TimeUnit::MICRO;
+ case flatbuf::TimeUnit_NANOSECOND:
+ return TimeUnit::NANO;
+ default:
+ break;
+ }
+ // cannot reach
+ return TimeUnit::SECOND;
+}
+
+static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
+ const std::vector<std::shared_ptr<Field>>& children, std::shared_ptr<DataType>* out) {
+ switch (type) {
+ case flatbuf::Type_NONE:
+ return Status::Invalid("Type metadata cannot be none");
+ case flatbuf::Type_Int:
+ return IntFromFlatbuffer(static_cast<const flatbuf::Int*>(type_data), out);
+ case flatbuf::Type_FloatingPoint:
+ return FloatFromFlatuffer(
+ static_cast<const flatbuf::FloatingPoint*>(type_data), out);
+ case flatbuf::Type_Binary:
+ *out = binary();
+ return Status::OK();
+ case flatbuf::Type_FixedWidthBinary: {
+ auto fw_binary = static_cast<const flatbuf::FixedWidthBinary*>(type_data);
+ *out = fixed_width_binary(fw_binary->byteWidth());
+ return Status::OK();
+ }
+ case flatbuf::Type_Utf8:
+ *out = utf8();
+ return Status::OK();
+ case flatbuf::Type_Bool:
+ *out = boolean();
+ return Status::OK();
+ case flatbuf::Type_Decimal:
+ return Status::NotImplemented("Decimal");
+ case flatbuf::Type_Date:
+ *out = date();
+ return Status::OK();
+ case flatbuf::Type_Time: {
+ auto time_type = static_cast<const flatbuf::Time*>(type_data);
+ *out = time(FromFlatbufferUnit(time_type->unit()));
+ return Status::OK();
+ }
+ case flatbuf::Type_Timestamp: {
+ auto ts_type = static_cast<const flatbuf::Timestamp*>(type_data);
+ *out = timestamp(FromFlatbufferUnit(ts_type->unit()));
+ return Status::OK();
+ }
+ case flatbuf::Type_Interval:
+ return Status::NotImplemented("Interval");
+ case flatbuf::Type_List:
+ if (children.size() != 1) {
+ return Status::Invalid("List must have exactly 1 child field");
+ }
+ *out = std::make_shared<ListType>(children[0]);
+ return Status::OK();
+ case flatbuf::Type_Struct_:
+ *out = std::make_shared<StructType>(children);
+ return Status::OK();
+ case flatbuf::Type_Union:
+ return UnionFromFlatbuffer(
+ static_cast<const flatbuf::Union*>(type_data), children, out);
+ default:
+ return Status::Invalid("Unrecognized type");
+ }
+}
+
+// TODO(wesm): Convert this to visitor pattern
+static Status TypeToFlatbuffer(FBB& fbb, const std::shared_ptr<DataType>& type,
+ std::vector<FieldOffset>* children, std::vector<VectorLayoutOffset>* layout,
+ flatbuf::Type* out_type, DictionaryMemo* dictionary_memo, Offset* offset) {
+ if (type->type == Type::DICTIONARY) {
+ // In this library, the dictionary "type" is a logical construct. Here we
+ // pass through to the value type, as we've already captured the index
+ // type in the DictionaryEncoding metadata in the parent field
+ const auto& dict_type = static_cast<const DictionaryType&>(*type);
+ return TypeToFlatbuffer(fbb, dict_type.dictionary()->type(), children, layout,
+ out_type, dictionary_memo, offset);
+ }
+
+ std::vector<BufferDescr> buffer_layout = type->GetBufferLayout();
+ for (const BufferDescr& descr : buffer_layout) {
+ flatbuf::VectorType vector_type;
+ switch (descr.type()) {
+ case BufferType::OFFSET:
+ vector_type = flatbuf::VectorType_OFFSET;
+ break;
+ case BufferType::DATA:
+ vector_type = flatbuf::VectorType_DATA;
+ break;
+ case BufferType::VALIDITY:
+ vector_type = flatbuf::VectorType_VALIDITY;
+ break;
+ case BufferType::TYPE:
+ vector_type = flatbuf::VectorType_TYPE;
+ break;
+ default:
+ vector_type = flatbuf::VectorType_DATA;
+ break;
+ }
+ auto offset = flatbuf::CreateVectorLayout(
+ fbb, static_cast<int16_t>(descr.bit_width()), vector_type);
+ layout->push_back(offset);
+ }
+
+ switch (type->type) {
+ case Type::BOOL:
+ *out_type = flatbuf::Type_Bool;
+ *offset = flatbuf::CreateBool(fbb).Union();
+ break;
+ case Type::UINT8:
+ INT_TO_FB_CASE(8, false);
+ case Type::INT8:
+ INT_TO_FB_CASE(8, true);
+ case Type::UINT16:
+ INT_TO_FB_CASE(16, false);
+ case Type::INT16:
+ INT_TO_FB_CASE(16, true);
+ case Type::UINT32:
+ INT_TO_FB_CASE(32, false);
+ case Type::INT32:
+ INT_TO_FB_CASE(32, true);
+ case Type::UINT64:
+ INT_TO_FB_CASE(64, false);
+ case Type::INT64:
+ INT_TO_FB_CASE(64, true);
+ case Type::FLOAT:
+ *out_type = flatbuf::Type_FloatingPoint;
+ *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_SINGLE);
+ break;
+ case Type::DOUBLE:
+ *out_type = flatbuf::Type_FloatingPoint;
+ *offset = FloatToFlatbuffer(fbb, flatbuf::Precision_DOUBLE);
+ break;
+ case Type::FIXED_WIDTH_BINARY: {
+ const auto& fw_type = static_cast<const FixedWidthBinaryType&>(*type);
+ *out_type = flatbuf::Type_FixedWidthBinary;
+ *offset = flatbuf::CreateFixedWidthBinary(fbb, fw_type.byte_width()).Union();
+ } break;
+ case Type::BINARY:
+ *out_type = flatbuf::Type_Binary;
+ *offset = flatbuf::CreateBinary(fbb).Union();
+ break;
+ case Type::STRING:
+ *out_type = flatbuf::Type_Utf8;
+ *offset = flatbuf::CreateUtf8(fbb).Union();
+ break;
+ case Type::DATE:
+ *out_type = flatbuf::Type_Date;
+ *offset = flatbuf::CreateDate(fbb).Union();
+ break;
+ case Type::TIME: {
+ const auto& time_type = static_cast<const TimeType&>(*type);
+ *out_type = flatbuf::Type_Time;
+ *offset = flatbuf::CreateTime(fbb, ToFlatbufferUnit(time_type.unit)).Union();
+ } break;
+ case Type::TIMESTAMP: {
+ const auto& ts_type = static_cast<const TimestampType&>(*type);
+ *out_type = flatbuf::Type_Timestamp;
+ *offset = flatbuf::CreateTimestamp(fbb, ToFlatbufferUnit(ts_type.unit)).Union();
+ } break;
+ case Type::LIST:
+ *out_type = flatbuf::Type_List;
+ return ListToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+ case Type::STRUCT:
+ *out_type = flatbuf::Type_Struct_;
+ return StructToFlatbuffer(fbb, type, children, dictionary_memo, offset);
+ case Type::UNION:
+ *out_type = flatbuf::Type_Union;
+ return UnionToFlatBuffer(fbb, type, children, dictionary_memo, offset);
+ default:
+ *out_type = flatbuf::Type_NONE; // Make clang-tidy happy
+ std::stringstream ss;
+ ss << "Unable to convert type: " << type->ToString() << std::endl;
+ return Status::NotImplemented(ss.str());
+ }
+ return Status::OK();
+}
+
+static DictionaryOffset GetDictionaryEncoding(
+ FBB& fbb, const DictionaryType& type, DictionaryMemo* memo) {
+ int64_t dictionary_id = memo->GetId(type.dictionary());
+
+ // We assume that the dictionary index type (as an integer) has already been
+ // validated elsewhere, and can safely assume we are dealing with signed
+ // integers
+ const auto& fw_index_type = static_cast<const FixedWidthType&>(*type.index_type());
+
+ auto index_type_offset = flatbuf::CreateInt(fbb, fw_index_type.bit_width(), true);
+
+ // TODO(wesm): ordered dictionaries
+ return flatbuf::CreateDictionaryEncoding(fbb, dictionary_id, index_type_offset);
+}
+
+static Status FieldToFlatbuffer(FBB& fbb, const std::shared_ptr<Field>& field,
+ DictionaryMemo* dictionary_memo, FieldOffset* offset) {
+ auto fb_name = fbb.CreateString(field->name);
+
+ flatbuf::Type type_enum;
+ Offset type_offset;
+ Offset type_layout;
+ std::vector<FieldOffset> children;
+ std::vector<VectorLayoutOffset> layout;
+
+ RETURN_NOT_OK(TypeToFlatbuffer(
+ fbb, field->type, &children, &layout, &type_enum, dictionary_memo, &type_offset));
+ auto fb_children = fbb.CreateVector(children);
+ auto fb_layout = fbb.CreateVector(layout);
+
+ DictionaryOffset dictionary = 0;
+ if (field->type->type == Type::DICTIONARY) {
+ dictionary = GetDictionaryEncoding(
+ fbb, static_cast<const DictionaryType&>(*field->type), dictionary_memo);
+ }
+
+ // TODO: produce the list of VectorTypes
+ *offset = flatbuf::CreateField(fbb, fb_name, field->nullable, type_enum, type_offset,
+ dictionary, fb_children, fb_layout);
+
+ return Status::OK();
+}
+
+static Status FieldFromFlatbuffer(const flatbuf::Field* field,
+ const DictionaryMemo& dictionary_memo, std::shared_ptr<Field>* out) {
+ std::shared_ptr<DataType> type;
+
+ const flatbuf::DictionaryEncoding* encoding = field->dictionary();
+
+ if (encoding == nullptr) {
+ // The field is not dictionary encoded. We must potentially visit its
+ // children to fully reconstruct the data type
+ auto children = field->children();
+ std::vector<std::shared_ptr<Field>> child_fields(children->size());
+ for (int i = 0; i < static_cast<int>(children->size()); ++i) {
+ RETURN_NOT_OK(
+ FieldFromFlatbuffer(children->Get(i), dictionary_memo, &child_fields[i]));
+ }
+ RETURN_NOT_OK(
+ TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
+ } else {
+ // The field is dictionary encoded. The type of the dictionary values has
+ // been determined elsewhere, and is stored in the DictionaryMemo. Here we
+ // construct the logical DictionaryType object
+
+ std::shared_ptr<Array> dictionary;
+ RETURN_NOT_OK(dictionary_memo.GetDictionary(encoding->id(), &dictionary));
+
+ std::shared_ptr<DataType> index_type;
+ RETURN_NOT_OK(IntFromFlatbuffer(encoding->indexType(), &index_type));
+ type = std::make_shared<DictionaryType>(index_type, dictionary);
+ }
+ *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
+ return Status::OK();
+}
+
+static Status FieldFromFlatbufferDictionary(
+ const flatbuf::Field* field, std::shared_ptr<Field>* out) {
+ // Need an empty memo to pass down for constructing children
+ DictionaryMemo dummy_memo;
+
+ // Any DictionaryEncoding set is ignored here
+
+ std::shared_ptr<DataType> type;
+ auto children = field->children();
+ std::vector<std::shared_ptr<Field>> child_fields(children->size());
+ for (int i = 0; i < static_cast<int>(children->size()); ++i) {
+ RETURN_NOT_OK(FieldFromFlatbuffer(children->Get(i), dummy_memo, &child_fields[i]));
+ }
+
+ RETURN_NOT_OK(
+ TypeFromFlatbuffer(field->type_type(), field->type(), child_fields, &type));
+
+ *out = std::make_shared<Field>(field->name()->str(), type, field->nullable());
+ return Status::OK();
+}
+
+// will return the endianness of the system we are running on
+// based the NUMPY_API function. See NOTICE.txt
+flatbuf::Endianness endianness() {
+ union {
+ uint32_t i;
+ char c[4];
+ } bint = {0x01020304};
+
+ return bint.c[0] == 1 ? flatbuf::Endianness_Big : flatbuf::Endianness_Little;
+}
+
+static Status SchemaToFlatbuffer(FBB& fbb, const Schema& schema,
+ DictionaryMemo* dictionary_memo, flatbuffers::Offset<flatbuf::Schema>* out) {
+ std::vector<FieldOffset> field_offsets;
+ for (int i = 0; i < schema.num_fields(); ++i) {
+ std::shared_ptr<Field> field = schema.field(i);
+ FieldOffset offset;
+ RETURN_NOT_OK(FieldToFlatbuffer(fbb, field, dictionary_memo, &offset));
+ field_offsets.push_back(offset);
+ }
+
+ *out = flatbuf::CreateSchema(fbb, endianness(), fbb.CreateVector(field_offsets));
+ return Status::OK();
+}
+
+static Status WriteFlatbufferBuilder(FBB& fbb, std::shared_ptr<Buffer>* out) {
+ int32_t size = fbb.GetSize();
+
+ auto result = std::make_shared<PoolBuffer>();
+ RETURN_NOT_OK(result->Resize(size));
+
+ uint8_t* dst = result->mutable_data();
+ memcpy(dst, fbb.GetBufferPointer(), size);
+ *out = result;
+ return Status::OK();
+}
+
+static Status WriteMessage(FBB& fbb, flatbuf::MessageHeader header_type,
+ flatbuffers::Offset<void> header, int64_t body_length, std::shared_ptr<Buffer>* out) {
+ auto message =
+ flatbuf::CreateMessage(fbb, kMetadataVersion, header_type, header, body_length);
+ fbb.Finish(message);
+ return WriteFlatbufferBuilder(fbb, out);
+}
+
+Status WriteSchemaMessage(
+ const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out) {
+ FBB fbb;
+ flatbuffers::Offset<flatbuf::Schema> fb_schema;
+ RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+ return WriteMessage(fbb, flatbuf::MessageHeader_Schema, fb_schema.Union(), 0, out);
+}
+
+using FieldNodeVector =
+ flatbuffers::Offset<flatbuffers::Vector<const flatbuf::FieldNode*>>;
+using LargeFieldNodeVector =
+ flatbuffers::Offset<flatbuffers::Vector<const flatbuf::LargeFieldNode*>>;
+using BufferVector = flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Buffer*>>;
+
+static Status WriteFieldNodes(
+ FBB& fbb, const std::vector<FieldMetadata>& nodes, FieldNodeVector* out) {
+ std::vector<flatbuf::FieldNode> fb_nodes;
+ fb_nodes.reserve(nodes.size());
+
+ for (size_t i = 0; i < nodes.size(); ++i) {
+ const FieldMetadata& node = nodes[i];
+ if (node.offset != 0) {
+ return Status::Invalid("Field metadata for IPC must have offset 0");
+ }
+ fb_nodes.emplace_back(
+ static_cast<int32_t>(node.length), static_cast<int32_t>(node.null_count));
+ }
+ *out = fbb.CreateVectorOfStructs(fb_nodes);
+ return Status::OK();
+}
+
+static Status WriteLargeFieldNodes(
+ FBB& fbb, const std::vector<FieldMetadata>& nodes, LargeFieldNodeVector* out) {
+ std::vector<flatbuf::LargeFieldNode> fb_nodes;
+ fb_nodes.reserve(nodes.size());
+
+ for (size_t i = 0; i < nodes.size(); ++i) {
+ const FieldMetadata& node = nodes[i];
+ if (node.offset != 0) {
+ return Status::Invalid("Field metadata for IPC must have offset 0");
+ }
+ fb_nodes.emplace_back(node.length, node.null_count);
+ }
+ *out = fbb.CreateVectorOfStructs(fb_nodes);
+ return Status::OK();
+}
+
+static Status WriteBuffers(
+ FBB& fbb, const std::vector<BufferMetadata>& buffers, BufferVector* out) {
+ std::vector<flatbuf::Buffer> fb_buffers;
+ fb_buffers.reserve(buffers.size());
+
+ for (size_t i = 0; i < buffers.size(); ++i) {
+ const BufferMetadata& buffer = buffers[i];
+ fb_buffers.emplace_back(buffer.page, buffer.offset, buffer.length);
+ }
+ *out = fbb.CreateVectorOfStructs(fb_buffers);
+ return Status::OK();
+}
+
+static Status MakeRecordBatch(FBB& fbb, int32_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ RecordBatchOffset* offset) {
+ FieldNodeVector fb_nodes;
+ BufferVector fb_buffers;
+
+ RETURN_NOT_OK(WriteFieldNodes(fbb, nodes, &fb_nodes));
+ RETURN_NOT_OK(WriteBuffers(fbb, buffers, &fb_buffers));
+
+ *offset = flatbuf::CreateRecordBatch(fbb, length, fb_nodes, fb_buffers);
+ return Status::OK();
+}
+
+static Status MakeLargeRecordBatch(FBB& fbb, int64_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ LargeRecordBatchOffset* offset) {
+ LargeFieldNodeVector fb_nodes;
+ BufferVector fb_buffers;
+
+ RETURN_NOT_OK(WriteLargeFieldNodes(fbb, nodes, &fb_nodes));
+ RETURN_NOT_OK(WriteBuffers(fbb, buffers, &fb_buffers));
+
+ *offset = flatbuf::CreateLargeRecordBatch(fbb, length, fb_nodes, fb_buffers);
+ return Status::OK();
+}
+
+Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ std::shared_ptr<Buffer>* out) {
+ FBB fbb;
+ RecordBatchOffset record_batch;
+ RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
+ return WriteMessage(
+ fbb, flatbuf::MessageHeader_RecordBatch, record_batch.Union(), body_length, out);
+}
+
+Status WriteLargeRecordBatchMessage(int64_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ std::shared_ptr<Buffer>* out) {
+ FBB fbb;
+ LargeRecordBatchOffset large_batch;
+ RETURN_NOT_OK(
+ MakeLargeRecordBatch(fbb, length, body_length, nodes, buffers, &large_batch));
+ return WriteMessage(fbb, flatbuf::MessageHeader_LargeRecordBatch, large_batch.Union(),
+ body_length, out);
+}
+
+Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ std::shared_ptr<Buffer>* out) {
+ FBB fbb;
+ RecordBatchOffset record_batch;
+ RETURN_NOT_OK(MakeRecordBatch(fbb, length, body_length, nodes, buffers, &record_batch));
+ auto dictionary_batch = flatbuf::CreateDictionaryBatch(fbb, id, record_batch).Union();
+ return WriteMessage(
+ fbb, flatbuf::MessageHeader_DictionaryBatch, dictionary_batch, body_length, out);
+}
+
+static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
+FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
+ std::vector<flatbuf::Block> fb_blocks;
+
+ for (const FileBlock& block : blocks) {
+ fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
+ }
+
+ return fbb.CreateVectorOfStructs(fb_blocks);
+}
+
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+ const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+ io::OutputStream* out) {
+ FBB fbb;
+
+ flatbuffers::Offset<flatbuf::Schema> fb_schema;
+ RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
+
+ auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
+ auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
+
+ auto footer = flatbuf::CreateFooter(
+ fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
+
+ fbb.Finish(footer);
+
+ int32_t size = fbb.GetSize();
+
+ return out->Write(fbb.GetBufferPointer(), size);
+}
+
// ----------------------------------------------------------------------
// Memoization data structure for handling shared dictionaries
@@ -158,7 +795,18 @@ int64_t Message::body_length() const {
// ----------------------------------------------------------------------
// SchemaMetadata
-class SchemaMetadata::SchemaMetadataImpl {
+class MessageHolder {
+ public:
+ void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
+ void set_buffer(const std::shared_ptr<Buffer>& buffer) { buffer_ = buffer; }
+
+ protected:
+ // Possible parents, owns the flatbuffer data
+ std::shared_ptr<Message> message_;
+ std::shared_ptr<Buffer> buffer_;
+};
+
+class SchemaMetadata::SchemaMetadataImpl : public MessageHolder {
public:
explicit SchemaMetadataImpl(const void* schema)
: schema_(static_cast<const flatbuf::Schema*>(schema)) {}
@@ -196,15 +844,19 @@ class SchemaMetadata::SchemaMetadataImpl {
const flatbuf::Schema* schema_;
};
-SchemaMetadata::SchemaMetadata(
- const std::shared_ptr<Message>& message, const void* flatbuf) {
- message_ = message;
- impl_.reset(new SchemaMetadataImpl(flatbuf));
+SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message)
+ : SchemaMetadata(message->impl_->header()) {
+ impl_->set_message(message);
}
-SchemaMetadata::SchemaMetadata(const std::shared_ptr<Message>& message) {
- message_ = message;
- impl_.reset(new SchemaMetadataImpl(message->impl_->header()));
+SchemaMetadata::SchemaMetadata(const void* header) {
+ impl_.reset(new SchemaMetadataImpl(header));
+}
+
+SchemaMetadata::SchemaMetadata(const std::shared_ptr<Buffer>& buffer, int64_t offset)
+ : SchemaMetadata(buffer->data() + offset) {
+ // Preserve ownership
+ impl_->set_buffer(buffer);
}
SchemaMetadata::~SchemaMetadata() {}
@@ -231,7 +883,7 @@ Status SchemaMetadata::GetSchema(
// ----------------------------------------------------------------------
// RecordBatchMetadata
-class RecordBatchMetadata::RecordBatchMetadataImpl {
+class RecordBatchMetadata::RecordBatchMetadataImpl : public MessageHolder {
public:
explicit RecordBatchMetadataImpl(const void* batch)
: batch_(static_cast<const flatbuf::RecordBatch*>(batch)) {
@@ -249,22 +901,14 @@ class RecordBatchMetadata::RecordBatchMetadataImpl {
int num_fields() const { return batch_->nodes()->size(); }
- void set_message(const std::shared_ptr<Message>& message) { message_ = message; }
-
- void set_buffer(const std::shared_ptr<Buffer>& buffer) { buffer_ = buffer; }
-
private:
const flatbuf::RecordBatch* batch_;
const flatbuffers::Vector<const flatbuf::FieldNode*>* nodes_;
const flatbuffers::Vector<const flatbuf::Buffer*>* buffers_;
-
- // Possible parents, owns the flatbuffer data
- std::shared_ptr<Message> message_;
- std::shared_ptr<Buffer> buffer_;
};
-RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message) {
- impl_.reset(new RecordBatchMetadataImpl(message->impl_->header()));
+RecordBatchMetadata::RecordBatchMetadata(const std::shared_ptr<Message>& message)
+ : RecordBatchMetadata(message->impl_->header()) {
impl_->set_message(message);
}
@@ -358,8 +1002,8 @@ const RecordBatchMetadata& DictionaryBatchMetadata::record_batch() const {
// ----------------------------------------------------------------------
// Conveniences
-Status ReadMessage(int64_t offset, int32_t metadata_length,
- io::RandomAccessFile* file, std::shared_ptr<Message>* message) {
+Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
+ std::shared_ptr<Message>* message) {
std::shared_ptr<Buffer> buffer;
RETURN_NOT_OK(file->ReadAt(offset, metadata_length, &buffer));
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/metadata.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata.h b/cpp/src/arrow/ipc/metadata.h
index 4eb0186..41e6c5e 100644
--- a/cpp/src/arrow/ipc/metadata.h
+++ b/cpp/src/arrow/ipc/metadata.h
@@ -107,10 +107,9 @@ class Message;
// Container for serialized Schema metadata contained in an IPC message
class ARROW_EXPORT SchemaMetadata {
public:
+ explicit SchemaMetadata(const void* header);
explicit SchemaMetadata(const std::shared_ptr<Message>& message);
-
- // Accepts an opaque flatbuffer pointer
- SchemaMetadata(const std::shared_ptr<Message>& message, const void* schema);
+ SchemaMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
~SchemaMetadata();
@@ -127,9 +126,6 @@ class ARROW_EXPORT SchemaMetadata {
const DictionaryMemo& dictionary_memo, std::shared_ptr<Schema>* out) const;
private:
- // Parent, owns the flatbuffer data
- std::shared_ptr<Message> message_;
-
class SchemaMetadataImpl;
std::unique_ptr<SchemaMetadataImpl> impl_;
@@ -145,8 +141,6 @@ struct ARROW_EXPORT BufferMetadata {
// Container for serialized record batch metadata contained in an IPC message
class ARROW_EXPORT RecordBatchMetadata {
public:
- // Instantiate from opaque pointer. Memory ownership must be preserved
- // elsewhere (e.g. in a dictionary batch)
explicit RecordBatchMetadata(const void* header);
explicit RecordBatchMetadata(const std::shared_ptr<Message>& message);
RecordBatchMetadata(const std::shared_ptr<Buffer>& message, int64_t offset);
@@ -218,8 +212,34 @@ class ARROW_EXPORT Message {
/// \param[in] file the seekable file interface to read from
/// \param[out] message the message read
/// \return Status success or failure
-Status ReadMessage(int64_t offset, int32_t metadata_length,
- io::RandomAccessFile* file, std::shared_ptr<Message>* message);
+Status ReadMessage(int64_t offset, int32_t metadata_length, io::RandomAccessFile* file,
+ std::shared_ptr<Message>* message);
+
+// Serialize arrow::Schema as a Flatbuffer
+//
+// \param[in] schema a Schema instance
+// \param[inout] dictionary_memo class for tracking dictionaries and assigning
+// dictionary ids
+// \param[out] out the serialized arrow::Buffer
+// \return Status outcome
+Status WriteSchemaMessage(
+ const Schema& schema, DictionaryMemo* dictionary_memo, std::shared_ptr<Buffer>* out);
+
+Status WriteRecordBatchMessage(int32_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ std::shared_ptr<Buffer>* out);
+
+Status WriteLargeRecordBatchMessage(int64_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ std::shared_ptr<Buffer>* out);
+
+Status WriteDictionaryMessage(int64_t id, int32_t length, int64_t body_length,
+ const std::vector<FieldMetadata>& nodes, const std::vector<BufferMetadata>& buffers,
+ std::shared_ptr<Buffer>* out);
+
+Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
+ const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
+ io::OutputStream* out);
} // namespace ipc
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/reader.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 9575364..a2b20a9 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -26,17 +26,115 @@
#include "arrow/buffer.h"
#include "arrow/io/interfaces.h"
#include "arrow/io/memory.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata-internal.h"
+#include "arrow/ipc/File_generated.h"
+#include "arrow/ipc/Message_generated.h"
#include "arrow/ipc/metadata.h"
#include "arrow/ipc/util.h"
+#include "arrow/schema.h"
#include "arrow/status.h"
+#include "arrow/table.h"
#include "arrow/util/logging.h"
namespace arrow {
+
+namespace flatbuf = org::apache::arrow::flatbuf;
+
namespace ipc {
// ----------------------------------------------------------------------
+// Record batch read path
+
+class IpcComponentSource : public ArrayComponentSource {
+ public:
+ IpcComponentSource(const RecordBatchMetadata& metadata, io::RandomAccessFile* file)
+ : metadata_(metadata), file_(file) {}
+
+ Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
+ BufferMetadata buffer_meta = metadata_.buffer(buffer_index);
+ if (buffer_meta.length == 0) {
+ *out = nullptr;
+ return Status::OK();
+ } else {
+ return file_->ReadAt(buffer_meta.offset, buffer_meta.length, out);
+ }
+ }
+
+ Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
+ // pop off a field
+ if (field_index >= metadata_.num_fields()) {
+ return Status::Invalid("Ran out of field metadata, likely malformed");
+ }
+ *metadata = metadata_.field(field_index);
+ return Status::OK();
+ }
+
+ private:
+ const RecordBatchMetadata& metadata_;
+ io::RandomAccessFile* file_;
+};
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+ const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
+ std::shared_ptr<RecordBatch>* out) {
+ return ReadRecordBatch(metadata, schema, kMaxNestingDepth, file, out);
+}
+
+static Status LoadRecordBatchFromSource(const std::shared_ptr<Schema>& schema,
+ int64_t num_rows, int max_recursion_depth, ArrayComponentSource* source,
+ std::shared_ptr<RecordBatch>* out) {
+ std::vector<std::shared_ptr<Array>> arrays(schema->num_fields());
+
+ ArrayLoaderContext context;
+ context.source = source;
+ context.field_index = 0;
+ context.buffer_index = 0;
+ context.max_recursion_depth = max_recursion_depth;
+
+ for (int i = 0; i < schema->num_fields(); ++i) {
+ RETURN_NOT_OK(LoadArray(schema->field(i)->type, &context, &arrays[i]));
+ }
+
+ *out = std::make_shared<RecordBatch>(schema, num_rows, arrays);
+ return Status::OK();
+}
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+ const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+ io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+ IpcComponentSource source(metadata, file);
+ return LoadRecordBatchFromSource(
+ schema, metadata.length(), max_recursion_depth, &source, out);
+}
+
+Status ReadDictionary(const DictionaryBatchMetadata& metadata,
+ const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
+ std::shared_ptr<Array>* out) {
+ int64_t id = metadata.id();
+ auto it = dictionary_types.find(id);
+ if (it == dictionary_types.end()) {
+ std::stringstream ss;
+ ss << "Do not have type metadata for dictionary with id: " << id;
+ return Status::KeyError(ss.str());
+ }
+
+ std::vector<std::shared_ptr<Field>> fields = {it->second};
+
+ // We need a schema for the record batch
+ auto dummy_schema = std::make_shared<Schema>(fields);
+
+ // The dictionary is embedded in a record batch with a single column
+ std::shared_ptr<RecordBatch> batch;
+ RETURN_NOT_OK(ReadRecordBatch(metadata.record_batch(), dummy_schema, file, &batch));
+
+ if (batch->num_columns() != 1) {
+ return Status::Invalid("Dictionary record batch must only contain one field");
+ }
+
+ *out = batch->column(0);
+ return Status::OK();
+}
+
+// ----------------------------------------------------------------------
// StreamReader implementation
static inline FileBlock FileBlockFromFlatbuffer(const flatbuf::Block* block) {
@@ -228,7 +326,7 @@ class FileReader::FileReaderImpl {
// TODO(wesm): Verify the footer
footer_ = flatbuf::GetFooter(footer_buffer_->data());
- schema_metadata_.reset(new SchemaMetadata(nullptr, footer_->schema()));
+ schema_metadata_.reset(new SchemaMetadata(footer_->schema()));
return Status::OK();
}
@@ -307,8 +405,7 @@ class FileReader::FileReaderImpl {
return schema_metadata_->GetSchema(*dictionary_memo_, &schema_);
}
- Status Open(
- const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
+ Status Open(const std::shared_ptr<io::RandomAccessFile>& file, int64_t footer_offset) {
file_ = file;
footer_offset_ = footer_offset;
RETURN_NOT_OK(ReadFooter());
@@ -371,5 +468,69 @@ Status FileReader::GetRecordBatch(int i, std::shared_ptr<RecordBatch>* batch) {
return impl_->GetRecordBatch(i, batch);
}
+// ----------------------------------------------------------------------
+// Read LargeRecordBatch
+
+class LargeRecordBatchSource : public ArrayComponentSource {
+ public:
+ LargeRecordBatchSource(
+ const flatbuf::LargeRecordBatch* metadata, io::RandomAccessFile* file)
+ : metadata_(metadata), file_(file) {}
+
+ Status GetBuffer(int buffer_index, std::shared_ptr<Buffer>* out) override {
+ if (buffer_index >= static_cast<int>(metadata_->buffers()->size())) {
+ return Status::Invalid("Ran out of buffer metadata, likely malformed");
+ }
+ const flatbuf::Buffer* buffer = metadata_->buffers()->Get(buffer_index);
+
+ if (buffer->length() == 0) {
+ *out = nullptr;
+ return Status::OK();
+ } else {
+ return file_->ReadAt(buffer->offset(), buffer->length(), out);
+ }
+ }
+
+ Status GetFieldMetadata(int field_index, FieldMetadata* metadata) override {
+ // pop off a field
+ if (field_index >= static_cast<int>(metadata_->nodes()->size())) {
+ return Status::Invalid("Ran out of field metadata, likely malformed");
+ }
+ const flatbuf::LargeFieldNode* node = metadata_->nodes()->Get(field_index);
+
+ metadata->length = node->length();
+ metadata->null_count = node->null_count();
+ metadata->offset = 0;
+ return Status::OK();
+ }
+
+ private:
+ const flatbuf::LargeRecordBatch* metadata_;
+ io::RandomAccessFile* file_;
+};
+
+Status ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
+ io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out) {
+ std::shared_ptr<Buffer> buffer;
+ RETURN_NOT_OK(file->Seek(offset));
+
+ RETURN_NOT_OK(file->Read(sizeof(int32_t), &buffer));
+ int32_t flatbuffer_size = *reinterpret_cast<const int32_t*>(buffer->data());
+
+ RETURN_NOT_OK(file->Read(flatbuffer_size, &buffer));
+ auto message = flatbuf::GetMessage(buffer->data());
+ auto batch = reinterpret_cast<const flatbuf::LargeRecordBatch*>(message->header());
+
+ // TODO(ARROW-388): The buffer offsets start at 0, so we must construct a
+ // RandomAccessFile according to that frame of reference
+ std::shared_ptr<Buffer> buffer_payload;
+ RETURN_NOT_OK(file->Read(message->bodyLength(), &buffer_payload));
+ io::BufferReader buffer_reader(buffer_payload);
+
+ LargeRecordBatchSource source(batch, &buffer_reader);
+ return LoadRecordBatchFromSource(
+ schema, batch->length(), kMaxNestingDepth, &source, out);
+}
+
} // namespace ipc
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/reader.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index ca91765..1c1314a 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -43,6 +43,20 @@ class RandomAccessFile;
namespace ipc {
+// Generic read functionsh; does not copy data if the input supports zero copy reads
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+ const std::shared_ptr<Schema>& schema, io::RandomAccessFile* file,
+ std::shared_ptr<RecordBatch>* out);
+
+Status ReadRecordBatch(const RecordBatchMetadata& metadata,
+ const std::shared_ptr<Schema>& schema, int max_recursion_depth,
+ io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+
+Status ReadDictionary(const DictionaryBatchMetadata& metadata,
+ const DictionaryTypeMap& dictionary_types, io::RandomAccessFile* file,
+ std::shared_ptr<Array>* out);
+
class ARROW_EXPORT StreamReader {
public:
~StreamReader();
@@ -106,6 +120,14 @@ class ARROW_EXPORT FileReader {
std::unique_ptr<FileReaderImpl> impl_;
};
+// ----------------------------------------------------------------------
+//
+
+/// EXPERIMENTAL: Read length-prefixed LargeRecordBatch metadata (64-bit array
+/// lengths) at offset and reconstruct RecordBatch
+Status ARROW_EXPORT ReadLargeRecordBatch(const std::shared_ptr<Schema>& schema,
+ int64_t offset, io::RandomAccessFile* file, std::shared_ptr<RecordBatch>* out);
+
} // namespace ipc
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index 66a5e09..ba203b0 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -103,7 +103,7 @@ Status MakeRandomListArray(const std::shared_ptr<Array>& child_array, int num_li
typedef Status MakeRecordBatch(std::shared_ptr<RecordBatch>* out);
Status MakeIntRecordBatch(std::shared_ptr<RecordBatch>* out) {
- const int length = 1000;
+ const int length = 10;
// Make the schema
auto f0 = field("f0", int32());
http://git-wip-us.apache.org/repos/asf/arrow/blob/df2220f3/cpp/src/arrow/ipc/writer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 58402b5..82c119e 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -17,28 +17,510 @@
#include "arrow/ipc/writer.h"
+#include <algorithm>
#include <cstdint>
#include <cstring>
+#include <limits>
#include <sstream>
#include <vector>
+#include "arrow/array.h"
#include "arrow/buffer.h"
#include "arrow/io/interfaces.h"
#include "arrow/io/memory.h"
-#include "arrow/ipc/adapter.h"
-#include "arrow/ipc/metadata-internal.h"
#include "arrow/ipc/metadata.h"
#include "arrow/ipc/util.h"
+#include "arrow/loader.h"
#include "arrow/memory_pool.h"
#include "arrow/schema.h"
#include "arrow/status.h"
#include "arrow/table.h"
+#include "arrow/type.h"
+#include "arrow/util/bit-util.h"
#include "arrow/util/logging.h"
namespace arrow {
namespace ipc {
// ----------------------------------------------------------------------
+// Record batch write path
+
+class RecordBatchWriter : public ArrayVisitor {
+ public:
+ RecordBatchWriter(
+ MemoryPool* pool, int64_t buffer_start_offset, int max_recursion_depth)
+ : pool_(pool),
+ max_recursion_depth_(max_recursion_depth),
+ buffer_start_offset_(buffer_start_offset) {
+ DCHECK_GT(max_recursion_depth, 0);
+ }
+
+ virtual ~RecordBatchWriter() = default;
+
+ virtual Status CheckArrayMetadata(const Array& arr) {
+ if (arr.length() > std::numeric_limits<int32_t>::max()) {
+ return Status::Invalid("Cannot write arrays larger than 2^31 - 1 in length");
+ }
+ return Status::OK();
+ }
+
+ Status VisitArray(const Array& arr) {
+ if (max_recursion_depth_ <= 0) {
+ return Status::Invalid("Max recursion depth reached");
+ }
+
+ RETURN_NOT_OK(CheckArrayMetadata(arr));
+
+ // push back all common elements
+ field_nodes_.emplace_back(arr.length(), arr.null_count(), 0);
+
+ if (arr.null_count() > 0) {
+ std::shared_ptr<Buffer> bitmap = arr.null_bitmap();
+
+ if (arr.offset() != 0) {
+ // With a sliced array / non-zero offset, we must copy the bitmap
+ RETURN_NOT_OK(
+ CopyBitmap(pool_, bitmap->data(), arr.offset(), arr.length(), &bitmap));
+ }
+
+ buffers_.push_back(bitmap);
+ } else {
+ // Push a dummy zero-length buffer, not to be copied
+ buffers_.push_back(std::make_shared<Buffer>(nullptr, 0));
+ }
+ return arr.Accept(this);
+ }
+
+ Status Assemble(const RecordBatch& batch, int64_t* body_length) {
+ if (field_nodes_.size() > 0) {
+ field_nodes_.clear();
+ buffer_meta_.clear();
+ buffers_.clear();
+ }
+
+ // Perform depth-first traversal of the row-batch
+ for (int i = 0; i < batch.num_columns(); ++i) {
+ RETURN_NOT_OK(VisitArray(*batch.column(i)));
+ }
+
+ // The position for the start of a buffer relative to the passed frame of
+ // reference. May be 0 or some other position in an address space
+ int64_t offset = buffer_start_offset_;
+
+ buffer_meta_.reserve(buffers_.size());
+
+ const int32_t kNoPageId = -1;
+
+ // Construct the buffer metadata for the record batch header
+ for (size_t i = 0; i < buffers_.size(); ++i) {
+ const Buffer* buffer = buffers_[i].get();
+ int64_t size = 0;
+ int64_t padding = 0;
+
+ // The buffer might be null if we are handling zero row lengths.
+ if (buffer) {
+ size = buffer->size();
+ padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+ }
+
+ // TODO(wesm): We currently have no notion of shared memory page id's,
+ // but we've included it in the metadata IDL for when we have it in the
+ // future. Use page = -1 for now
+ //
+ // Note that page ids are a bespoke notion for Arrow and not a feature we
+ // are using from any OS-level shared memory. The thought is that systems
+ // may (in the future) associate integer page id's with physical memory
+ // pages (according to whatever is the desired shared memory mechanism)
+ buffer_meta_.push_back({kNoPageId, offset, size + padding});
+ offset += size + padding;
+ }
+
+ *body_length = offset - buffer_start_offset_;
+ DCHECK(BitUtil::IsMultipleOf64(*body_length));
+
+ return Status::OK();
+ }
+
+ // Override this for writing dictionary metadata
+ virtual Status WriteMetadataMessage(
+ int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) {
+ return WriteRecordBatchMessage(
+ static_cast<int32_t>(num_rows), body_length, field_nodes_, buffer_meta_, out);
+ }
+
+ Status WriteMetadata(int64_t num_rows, int64_t body_length, io::OutputStream* dst,
+ int32_t* metadata_length) {
+ // Now that we have computed the locations of all of the buffers in shared
+ // memory, the data header can be converted to a flatbuffer and written out
+ //
+ // Note: The memory written here is prefixed by the size of the flatbuffer
+ // itself as an int32_t.
+ std::shared_ptr<Buffer> metadata_fb;
+ RETURN_NOT_OK(WriteMetadataMessage(num_rows, body_length, &metadata_fb));
+
+ // Need to write 4 bytes (metadata size), the metadata, plus padding to
+ // end on an 8-byte offset
+ int64_t start_offset;
+ RETURN_NOT_OK(dst->Tell(&start_offset));
+
+ int32_t padded_metadata_length = static_cast<int32_t>(metadata_fb->size()) + 4;
+ const int32_t remainder =
+ (padded_metadata_length + static_cast<int32_t>(start_offset)) % 8;
+ if (remainder != 0) { padded_metadata_length += 8 - remainder; }
+
+ // The returned metadata size includes the length prefix, the flatbuffer,
+ // plus padding
+ *metadata_length = padded_metadata_length;
+
+ // Write the flatbuffer size prefix including padding
+ int32_t flatbuffer_size = padded_metadata_length - 4;
+ RETURN_NOT_OK(
+ dst->Write(reinterpret_cast<const uint8_t*>(&flatbuffer_size), sizeof(int32_t)));
+
+ // Write the flatbuffer
+ RETURN_NOT_OK(dst->Write(metadata_fb->data(), metadata_fb->size()));
+
+ // Write any padding
+ int32_t padding =
+ padded_metadata_length - static_cast<int32_t>(metadata_fb->size()) - 4;
+ if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
+
+ return Status::OK();
+ }
+
+ Status Write(const RecordBatch& batch, io::OutputStream* dst, int32_t* metadata_length,
+ int64_t* body_length) {
+ RETURN_NOT_OK(Assemble(batch, body_length));
+
+#ifndef NDEBUG
+ int64_t start_position, current_position;
+ RETURN_NOT_OK(dst->Tell(&start_position));
+#endif
+
+ RETURN_NOT_OK(WriteMetadata(batch.num_rows(), *body_length, dst, metadata_length));
+
+#ifndef NDEBUG
+ RETURN_NOT_OK(dst->Tell(¤t_position));
+ DCHECK(BitUtil::IsMultipleOf8(current_position));
+#endif
+
+ // Now write the buffers
+ for (size_t i = 0; i < buffers_.size(); ++i) {
+ const Buffer* buffer = buffers_[i].get();
+ int64_t size = 0;
+ int64_t padding = 0;
+
+ // The buffer might be null if we are handling zero row lengths.
+ if (buffer) {
+ size = buffer->size();
+ padding = BitUtil::RoundUpToMultipleOf64(size) - size;
+ }
+
+ if (size > 0) { RETURN_NOT_OK(dst->Write(buffer->data(), size)); }
+
+ if (padding > 0) { RETURN_NOT_OK(dst->Write(kPaddingBytes, padding)); }
+ }
+
+#ifndef NDEBUG
+ RETURN_NOT_OK(dst->Tell(¤t_position));
+ DCHECK(BitUtil::IsMultipleOf8(current_position));
+#endif
+
+ return Status::OK();
+ }
+
+ Status GetTotalSize(const RecordBatch& batch, int64_t* size) {
+ // emulates the behavior of Write without actually writing
+ int32_t metadata_length = 0;
+ int64_t body_length = 0;
+ MockOutputStream dst;
+ RETURN_NOT_OK(Write(batch, &dst, &metadata_length, &body_length));
+ *size = dst.GetExtentBytesWritten();
+ return Status::OK();
+ }
+
+ protected:
+ template <typename ArrayType>
+ Status VisitFixedWidth(const ArrayType& array) {
+ std::shared_ptr<Buffer> data_buffer = array.data();
+
+ if (array.offset() != 0) {
+ // Non-zero offset, slice the buffer
+ const auto& fw_type = static_cast<const FixedWidthType&>(*array.type());
+ const int type_width = fw_type.bit_width() / 8;
+ const int64_t byte_offset = array.offset() * type_width;
+
+ // Send padding if it's available
+ const int64_t buffer_length =
+ std::min(BitUtil::RoundUpToMultipleOf64(array.length() * type_width),
+ data_buffer->size() - byte_offset);
+ data_buffer = SliceBuffer(data_buffer, byte_offset, buffer_length);
+ }
+ buffers_.push_back(data_buffer);
+ return Status::OK();
+ }
+
+ template <typename ArrayType>
+ Status GetZeroBasedValueOffsets(
+ const ArrayType& array, std::shared_ptr<Buffer>* value_offsets) {
+ // Share slicing logic between ListArray and BinaryArray
+
+ auto offsets = array.value_offsets();
+
+ if (array.offset() != 0) {
+ // If we have a non-zero offset, then the value offsets do not start at
+ // zero. We must a) create a new offsets array with shifted offsets and
+ // b) slice the values array accordingly
+
+ std::shared_ptr<MutableBuffer> shifted_offsets;
+ RETURN_NOT_OK(AllocateBuffer(
+ pool_, sizeof(int32_t) * (array.length() + 1), &shifted_offsets));
+
+ int32_t* dest_offsets = reinterpret_cast<int32_t*>(shifted_offsets->mutable_data());
+ const int32_t start_offset = array.value_offset(0);
+
+ for (int i = 0; i < array.length(); ++i) {
+ dest_offsets[i] = array.value_offset(i) - start_offset;
+ }
+ // Final offset
+ dest_offsets[array.length()] = array.value_offset(array.length()) - start_offset;
+ offsets = shifted_offsets;
+ }
+
+ *value_offsets = offsets;
+ return Status::OK();
+ }
+
+ Status VisitBinary(const BinaryArray& array) {
+ std::shared_ptr<Buffer> value_offsets;
+ RETURN_NOT_OK(GetZeroBasedValueOffsets<BinaryArray>(array, &value_offsets));
+ auto data = array.data();
+
+ if (array.offset() != 0) {
+ // Slice the data buffer to include only the range we need now
+ data = SliceBuffer(data, array.value_offset(0), array.value_offset(array.length()));
+ }
+
+ buffers_.push_back(value_offsets);
+ buffers_.push_back(data);
+ return Status::OK();
+ }
+
+ Status Visit(const FixedWidthBinaryArray& array) override {
+ auto data = array.data();
+ int32_t width = array.byte_width();
+
+ if (array.offset() != 0) {
+ data = SliceBuffer(data, array.offset() * width, width * array.length());
+ }
+ buffers_.push_back(data);
+ return Status::OK();
+ }
+
+ Status Visit(const BooleanArray& array) override {
+ buffers_.push_back(array.data());
+ return Status::OK();
+ }
+
+#define VISIT_FIXED_WIDTH(TYPE) \
+ Status Visit(const TYPE& array) override { return VisitFixedWidth<TYPE>(array); }
+
+ VISIT_FIXED_WIDTH(Int8Array);
+ VISIT_FIXED_WIDTH(Int16Array);
+ VISIT_FIXED_WIDTH(Int32Array);
+ VISIT_FIXED_WIDTH(Int64Array);
+ VISIT_FIXED_WIDTH(UInt8Array);
+ VISIT_FIXED_WIDTH(UInt16Array);
+ VISIT_FIXED_WIDTH(UInt32Array);
+ VISIT_FIXED_WIDTH(UInt64Array);
+ VISIT_FIXED_WIDTH(HalfFloatArray);
+ VISIT_FIXED_WIDTH(FloatArray);
+ VISIT_FIXED_WIDTH(DoubleArray);
+ VISIT_FIXED_WIDTH(DateArray);
+ VISIT_FIXED_WIDTH(Date32Array);
+ VISIT_FIXED_WIDTH(TimeArray);
+ VISIT_FIXED_WIDTH(TimestampArray);
+
+#undef VISIT_FIXED_WIDTH
+
+ Status Visit(const StringArray& array) override { return VisitBinary(array); }
+
+ Status Visit(const BinaryArray& array) override { return VisitBinary(array); }
+
+ Status Visit(const ListArray& array) override {
+ std::shared_ptr<Buffer> value_offsets;
+ RETURN_NOT_OK(GetZeroBasedValueOffsets<ListArray>(array, &value_offsets));
+ buffers_.push_back(value_offsets);
+
+ --max_recursion_depth_;
+ std::shared_ptr<Array> values = array.values();
+
+ if (array.offset() != 0) {
+ // For non-zero offset, we slice the values array accordingly
+ const int32_t offset = array.value_offset(0);
+ const int32_t length = array.value_offset(array.length()) - offset;
+ values = values->Slice(offset, length);
+ }
+ RETURN_NOT_OK(VisitArray(*values));
+ ++max_recursion_depth_;
+ return Status::OK();
+ }
+
+ Status Visit(const StructArray& array) override {
+ --max_recursion_depth_;
+ for (std::shared_ptr<Array> field : array.fields()) {
+ if (array.offset() != 0) {
+ // If offset is non-zero, slice the child array
+ field = field->Slice(array.offset(), array.length());
+ }
+ RETURN_NOT_OK(VisitArray(*field));
+ }
+ ++max_recursion_depth_;
+ return Status::OK();
+ }
+
+ Status Visit(const UnionArray& array) override {
+ auto type_ids = array.type_ids();
+ if (array.offset() != 0) {
+ type_ids = SliceBuffer(type_ids, array.offset() * sizeof(UnionArray::type_id_t),
+ array.length() * sizeof(UnionArray::type_id_t));
+ }
+
+ buffers_.push_back(type_ids);
+
+ --max_recursion_depth_;
+ if (array.mode() == UnionMode::DENSE) {
+ const auto& type = static_cast<const UnionType&>(*array.type());
+ auto value_offsets = array.value_offsets();
+
+ // The Union type codes are not necessary 0-indexed
+ uint8_t max_code = 0;
+ for (uint8_t code : type.type_codes) {
+ if (code > max_code) { max_code = code; }
+ }
+
+ // Allocate an array of child offsets. Set all to -1 to indicate that we
+ // haven't observed a first occurrence of a particular child yet
+ std::vector<int32_t> child_offsets(max_code + 1);
+ std::vector<int32_t> child_lengths(max_code + 1, 0);
+
+ if (array.offset() != 0) {
+ // This is an unpleasant case. Because the offsets are different for
+ // each child array, when we have a sliced array, we need to "rebase"
+ // the value_offsets for each array
+
+ const int32_t* unshifted_offsets = array.raw_value_offsets();
+ const uint8_t* type_ids = array.raw_type_ids();
+
+ // Allocate the shifted offsets
+ std::shared_ptr<MutableBuffer> shifted_offsets_buffer;
+ RETURN_NOT_OK(AllocateBuffer(
+ pool_, array.length() * sizeof(int32_t), &shifted_offsets_buffer));
+ int32_t* shifted_offsets =
+ reinterpret_cast<int32_t*>(shifted_offsets_buffer->mutable_data());
+
+ for (int64_t i = 0; i < array.length(); ++i) {
+ const uint8_t code = type_ids[i];
+ int32_t shift = child_offsets[code];
+ if (shift == -1) { child_offsets[code] = shift = unshifted_offsets[i]; }
+ shifted_offsets[i] = unshifted_offsets[i] - shift;
+
+ // Update the child length to account for observed value
+ ++child_lengths[code];
+ }
+
+ value_offsets = shifted_offsets_buffer;
+ }
+ buffers_.push_back(value_offsets);
+
+ // Visit children and slice accordingly
+ for (int i = 0; i < type.num_children(); ++i) {
+ std::shared_ptr<Array> child = array.child(i);
+ if (array.offset() != 0) {
+ const uint8_t code = type.type_codes[i];
+ child = child->Slice(child_offsets[code], child_lengths[code]);
+ }
+ RETURN_NOT_OK(VisitArray(*child));
+ }
+ } else {
+ for (std::shared_ptr<Array> child : array.children()) {
+ // Sparse union, slicing is simpler
+ if (array.offset() != 0) {
+ // If offset is non-zero, slice the child array
+ child = child->Slice(array.offset(), array.length());
+ }
+ RETURN_NOT_OK(VisitArray(*child));
+ }
+ }
+ ++max_recursion_depth_;
+ return Status::OK();
+ }
+
+ Status Visit(const DictionaryArray& array) override {
+ // Dictionary written out separately. Slice offset contained in the indices
+ return array.indices()->Accept(this);
+ }
+
+ // In some cases, intermediate buffers may need to be allocated (with sliced arrays)
+ MemoryPool* pool_;
+
+ std::vector<FieldMetadata> field_nodes_;
+ std::vector<BufferMetadata> buffer_meta_;
+ std::vector<std::shared_ptr<Buffer>> buffers_;
+
+ int64_t max_recursion_depth_;
+ int64_t buffer_start_offset_;
+};
+
+class DictionaryWriter : public RecordBatchWriter {
+ public:
+ using RecordBatchWriter::RecordBatchWriter;
+
+ Status WriteMetadataMessage(
+ int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
+ return WriteDictionaryMessage(dictionary_id_, static_cast<int32_t>(num_rows),
+ body_length, field_nodes_, buffer_meta_, out);
+ }
+
+ Status Write(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+ io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length) {
+ dictionary_id_ = dictionary_id;
+
+ // Make a dummy record batch. A bit tedious as we have to make a schema
+ std::vector<std::shared_ptr<Field>> fields = {
+ arrow::field("dictionary", dictionary->type())};
+ auto schema = std::make_shared<Schema>(fields);
+ RecordBatch batch(schema, dictionary->length(), {dictionary});
+
+ return RecordBatchWriter::Write(batch, dst, metadata_length, body_length);
+ }
+
+ private:
+ // TODO(wesm): Setting this in Write is a bit unclean, but it works
+ int64_t dictionary_id_;
+};
+
+Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+ io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+ MemoryPool* pool, int max_recursion_depth) {
+ RecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
+ return writer.Write(batch, dst, metadata_length, body_length);
+}
+
+Status WriteDictionary(int64_t dictionary_id, const std::shared_ptr<Array>& dictionary,
+ int64_t buffer_start_offset, io::OutputStream* dst, int32_t* metadata_length,
+ int64_t* body_length, MemoryPool* pool) {
+ DictionaryWriter writer(pool, buffer_start_offset, kMaxNestingDepth);
+ return writer.Write(dictionary_id, dictionary, dst, metadata_length, body_length);
+}
+
+Status GetRecordBatchSize(const RecordBatch& batch, int64_t* size) {
+ RecordBatchWriter writer(default_memory_pool(), 0, kMaxNestingDepth);
+ RETURN_NOT_OK(writer.GetTotalSize(batch, size));
+ return Status::OK();
+}
+
+// ----------------------------------------------------------------------
// Stream writer implementation
class StreamWriter::StreamWriterImpl {
@@ -199,38 +681,6 @@ Status StreamWriter::Close() {
// ----------------------------------------------------------------------
// File writer implementation
-static flatbuffers::Offset<flatbuffers::Vector<const flatbuf::Block*>>
-FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
- std::vector<flatbuf::Block> fb_blocks;
-
- for (const FileBlock& block : blocks) {
- fb_blocks.emplace_back(block.offset, block.metadata_length, block.body_length);
- }
-
- return fbb.CreateVectorOfStructs(fb_blocks);
-}
-
-Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
- const std::vector<FileBlock>& record_batches, DictionaryMemo* dictionary_memo,
- io::OutputStream* out) {
- FBB fbb;
-
- flatbuffers::Offset<flatbuf::Schema> fb_schema;
- RETURN_NOT_OK(SchemaToFlatbuffer(fbb, schema, dictionary_memo, &fb_schema));
-
- auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
- auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
-
- auto footer = flatbuf::CreateFooter(
- fbb, kMetadataVersion, fb_schema, fb_dictionaries, fb_record_batches);
-
- fbb.Finish(footer);
-
- int32_t size = fbb.GetSize();
-
- return out->Write(fbb.GetBufferPointer(), size);
-}
-
class FileWriter::FileWriterImpl : public StreamWriter::StreamWriterImpl {
public:
using BASE = StreamWriter::StreamWriterImpl;
@@ -283,5 +733,31 @@ Status FileWriter::Close() {
return impl_->Close();
}
+// ----------------------------------------------------------------------
+// Write record batches with 64-bit size metadata
+
+class LargeRecordBatchWriter : public RecordBatchWriter {
+ public:
+ using RecordBatchWriter::RecordBatchWriter;
+
+ Status CheckArrayMetadata(const Array& arr) override {
+ // No < INT32_MAX length check
+ return Status::OK();
+ }
+
+ Status WriteMetadataMessage(
+ int64_t num_rows, int64_t body_length, std::shared_ptr<Buffer>* out) override {
+ return WriteLargeRecordBatchMessage(
+ num_rows, body_length, field_nodes_, buffer_meta_, out);
+ }
+};
+
+Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
+ io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
+ MemoryPool* pool, int max_recursion_depth) {
+ LargeRecordBatchWriter writer(pool, buffer_start_offset, max_recursion_depth);
+ return writer.Write(batch, dst, metadata_length, body_length);
+}
+
} // namespace ipc
} // namespace arrow