You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/04/09 17:56:20 UTC
arrow git commit: ARROW-788: [C++] Align WriteTensor message
Repository: arrow
Updated Branches:
refs/heads/master 739ed8202 -> b0863cb63
ARROW-788: [C++] Align WriteTensor message
Author: Philipp Moritz <pc...@gmail.com>
Closes #512 from pcmoritz/tensor-alignment and squashes the following commits:
fd20f05 [Philipp Moritz] align WriteTensor to 8 bytes
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b0863cb6
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b0863cb6
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b0863cb6
Branch: refs/heads/master
Commit: b0863cb63d62ae7c4a429164e5a2e350d3c1f21a
Parents: 739ed82
Author: Philipp Moritz <pc...@gmail.com>
Authored: Sun Apr 9 13:56:14 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Sun Apr 9 13:56:14 2017 -0400
----------------------------------------------------------------------
cpp/src/arrow/ipc/reader.cc | 2 ++
cpp/src/arrow/ipc/writer.cc | 11 +++++++++++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/b0863cb6/cpp/src/arrow/ipc/reader.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 55f632f..a7c4f04 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -495,6 +495,8 @@ Status ReadRecordBatch(const std::shared_ptr<Schema>& schema, int64_t offset,
Status ReadTensor(
int64_t offset, io::RandomAccessFile* file, std::shared_ptr<Tensor>* out) {
+ // Respect alignment of Tensor messages (see WriteTensor)
+ offset = PaddedLength(offset);
std::shared_ptr<Message> message;
std::shared_ptr<Buffer> data;
RETURN_NOT_OK(ReadContiguousPayload(offset, file, &message, &data));
http://git-wip-us.apache.org/repos/asf/arrow/blob/b0863cb6/cpp/src/arrow/ipc/writer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 5330206..9305567 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -470,6 +470,16 @@ class DictionaryWriter : public RecordBatchWriter {
int64_t dictionary_id_;
};
+// Adds padding bytes if necessary to ensure all memory blocks are written on
+// 8-byte boundaries.
+Status AlignStreamPosition(io::OutputStream* stream) {
+ int64_t position;
+ RETURN_NOT_OK(stream->Tell(&position));
+ int64_t remainder = PaddedLength(position) - position;
+ if (remainder > 0) { return stream->Write(kPaddingBytes, remainder); }
+ return Status::OK();
+}
+
Status WriteRecordBatch(const RecordBatch& batch, int64_t buffer_start_offset,
io::OutputStream* dst, int32_t* metadata_length, int64_t* body_length,
MemoryPool* pool, int max_recursion_depth, bool allow_64bit) {
@@ -486,6 +496,7 @@ Status WriteLargeRecordBatch(const RecordBatch& batch, int64_t buffer_start_offs
Status WriteTensor(const Tensor& tensor, io::OutputStream* dst, int32_t* metadata_length,
int64_t* body_length) {
+ RETURN_NOT_OK(AlignStreamPosition(dst));
std::shared_ptr<Buffer> metadata;
RETURN_NOT_OK(WriteTensorMessage(tensor, 0, &metadata));
RETURN_NOT_OK(WriteMessage(*metadata, dst, metadata_length));