You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2020/04/04 23:02:52 UTC
[arrow] branch master updated: ARROW-6837: [C++] Add APIs to read
and write "custom_metadata" field of IPC file footer
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 9f99a98 ARROW-6837: [C++] Add APIs to read and write "custom_metadata" field of IPC file footer
9f99a98 is described below
commit 9f99a989853694b20d85130d53f865288206ffc3
Author: John Muehlhausen <jm...@kershnertrading.com>
AuthorDate: Sat Apr 4 18:02:36 2020 -0500
ARROW-6837: [C++] Add APIs to read and write "custom_metadata" field of IPC file footer
Closes #6838 from wesm/ARROW-6837
Lead-authored-by: John Muehlhausen <jm...@kershnertrading.com>
Co-authored-by: Wes McKinney <we...@apache.org>
Signed-off-by: Wes McKinney <we...@apache.org>
---
cpp/src/arrow/ipc/metadata_internal.cc | 8 ++++++--
cpp/src/arrow/ipc/metadata_internal.h | 1 +
cpp/src/arrow/ipc/read_write_test.cc | 29 +++++++++++++++++++++++++++--
cpp/src/arrow/ipc/reader.cc | 8 ++++++++
cpp/src/arrow/ipc/reader.h | 4 ++++
cpp/src/arrow/ipc/writer.cc | 13 +++++++++----
cpp/src/arrow/ipc/writer.h | 6 ++++--
7 files changed, 59 insertions(+), 10 deletions(-)
diff --git a/cpp/src/arrow/ipc/metadata_internal.cc b/cpp/src/arrow/ipc/metadata_internal.cc
index 365a6cc..8287b06 100644
--- a/cpp/src/arrow/ipc/metadata_internal.cc
+++ b/cpp/src/arrow/ipc/metadata_internal.cc
@@ -1184,6 +1184,7 @@ FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
const std::vector<FileBlock>& record_batches,
+ const std::shared_ptr<const KeyValueMetadata>& metadata,
io::OutputStream* out) {
FBB fbb;
@@ -1208,8 +1209,11 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
- auto footer = flatbuf::CreateFooter(fbb, kCurrentMetadataVersion, fb_schema,
- fb_dictionaries, fb_record_batches);
+ auto fb_custom_metadata = SerializeCustomMetadata(fbb, metadata);
+
+ auto footer =
+ flatbuf::CreateFooter(fbb, kCurrentMetadataVersion, fb_schema, fb_dictionaries,
+ fb_record_batches, fb_custom_metadata);
fbb.Finish(footer);
int32_t size = fbb.GetSize();
diff --git a/cpp/src/arrow/ipc/metadata_internal.h b/cpp/src/arrow/ipc/metadata_internal.h
index 8706fee..617f9f4 100644
--- a/cpp/src/arrow/ipc/metadata_internal.h
+++ b/cpp/src/arrow/ipc/metadata_internal.h
@@ -190,6 +190,7 @@ Result<std::shared_ptr<Buffer>> WriteSparseTensorMessage(
Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
const std::vector<FileBlock>& record_batches,
+ const std::shared_ptr<const KeyValueMetadata>& metadata,
io::OutputStream* out);
Status WriteDictionaryMessage(
diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc
index 84b834f..d1e39d8 100644
--- a/cpp/src/arrow/ipc/read_write_test.cc
+++ b/cpp/src/arrow/ipc/read_write_test.cc
@@ -789,12 +789,13 @@ TEST_F(RecursionLimits, StressLimit) {
#endif // !defined(_WIN32) || defined(NDEBUG)
struct FileWriterHelper {
- Status Init(const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options) {
+ Status Init(const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options,
+ const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr) {
num_batches_written_ = 0;
RETURN_NOT_OK(AllocateResizableBuffer(0, &buffer_));
sink_.reset(new io::BufferOutputStream(buffer_));
- ARROW_ASSIGN_OR_RAISE(writer_, NewFileWriter(sink_.get(), schema, options));
+ ARROW_ASSIGN_OR_RAISE(writer_, NewFileWriter(sink_.get(), schema, options, metadata));
return Status::OK();
}
@@ -836,6 +837,13 @@ struct FileWriterHelper {
return Status::OK();
}
+ Result<std::shared_ptr<const KeyValueMetadata>> ReadFooterMetadata() {
+ auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+ ARROW_ASSIGN_OR_RAISE(auto reader,
+ RecordBatchFileReader::Open(buf_reader.get(), footer_offset_));
+ return reader->metadata();
+ }
+
std::shared_ptr<ResizableBuffer> buffer_;
std::unique_ptr<io::BufferOutputStream> sink_;
std::shared_ptr<RecordBatchWriter> writer_;
@@ -1090,6 +1098,23 @@ INSTANTIATE_TEST_SUITE_P(GenericIpcRoundTripTests, TestIpcRoundTrip, BATCH_CASES
INSTANTIATE_TEST_SUITE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
INSTANTIATE_TEST_SUITE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
+TEST(TestIpcFileFormat, FooterMetaData) {
+ // ARROW-6837
+ std::shared_ptr<RecordBatch> batch;
+ ASSERT_OK(MakeIntRecordBatch(&batch));
+
+ auto metadata = key_value_metadata({"ARROW:example", "ARROW:example2"},
+ {"something something", "something something2"});
+
+ FileWriterHelper helper;
+ ASSERT_OK(helper.Init(batch->schema(), IpcWriteOptions::Defaults(), metadata));
+ ASSERT_OK(helper.WriteBatch(batch));
+ ASSERT_OK(helper.Finish());
+
+ ASSERT_OK_AND_ASSIGN(auto out_metadata, helper.ReadFooterMetadata());
+ ASSERT_TRUE(out_metadata->Equals(*metadata));
+}
+
// This test uses uninitialized memory
#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 8105987..9cefb51 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -799,6 +799,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
std::shared_ptr<Schema> schema() const override { return schema_; }
+ std::shared_ptr<const KeyValueMetadata> metadata() const override { return metadata_; }
+
private:
FileBlock GetRecordBatchBlock(int i) const {
return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
@@ -874,6 +876,11 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
}
footer_ = flatbuf::GetFooter(data);
+ auto fb_metadata = footer_->custom_metadata();
+ if (fb_metadata != nullptr) {
+ RETURN_NOT_OK(internal::GetKeyValueMetadata(fb_metadata, &metadata_));
+ }
+
return Status::OK();
}
@@ -894,6 +901,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
// Footer metadata
std::shared_ptr<Buffer> footer_buffer_;
const flatbuf::Footer* footer_;
+ std::shared_ptr<const KeyValueMetadata> metadata_;
bool read_dictionaries_ = false;
DictionaryMemo dictionary_memo_;
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index dd2ebd4..903b952 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -177,6 +177,10 @@ class ARROW_EXPORT RecordBatchFileReader {
/// \brief Return the metadata version from the file metadata
virtual MetadataVersion version() const = 0;
+ /// \brief Return the contents of the custom_metadata field from the file's
+ /// Footer
+ virtual std::shared_ptr<const KeyValueMetadata> metadata() const = 0;
+
/// \brief Read a particular record batch from the file. Does not copy memory
/// if the input source supports zero-copy.
///
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index ccec08a..ad27fb8 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -1102,8 +1102,9 @@ class PayloadStreamWriter : public IpcPayloadWriter, protected StreamBookKeeper
class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBookKeeper {
public:
PayloadFileWriter(const IpcWriteOptions& options, const std::shared_ptr<Schema>& schema,
+ const std::shared_ptr<const KeyValueMetadata>& metadata,
io::OutputStream* sink)
- : StreamBookKeeper(options, sink), schema_(schema) {}
+ : StreamBookKeeper(options, sink), schema_(schema), metadata_(metadata) {}
~PayloadFileWriter() override = default;
@@ -1153,7 +1154,8 @@ class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBoo
// Write file footer
RETURN_NOT_OK(UpdatePosition());
int64_t initial_position = position_;
- RETURN_NOT_OK(WriteFileFooter(*schema_, dictionaries_, record_batches_, sink_));
+ RETURN_NOT_OK(
+ WriteFileFooter(*schema_, dictionaries_, record_batches_, metadata_, sink_));
// Write footer length
RETURN_NOT_OK(UpdatePosition());
@@ -1170,6 +1172,7 @@ class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBoo
protected:
std::shared_ptr<Schema> schema_;
+ std::shared_ptr<const KeyValueMetadata> metadata_;
std::vector<FileBlock> dictionaries_;
std::vector<FileBlock> record_batches_;
};
@@ -1186,9 +1189,11 @@ Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
- const IpcWriteOptions& options) {
+ const IpcWriteOptions& options,
+ const std::shared_ptr<const KeyValueMetadata>& metadata) {
return std::make_shared<internal::IpcFormatWriter>(
- ::arrow::internal::make_unique<internal::PayloadFileWriter>(options, schema, sink),
+ ::arrow::internal::make_unique<internal::PayloadFileWriter>(options, schema,
+ metadata, sink),
schema, options);
}
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 2d47a37..f2b057e 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -99,12 +99,14 @@ Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
///
/// \param[in] sink output stream to write to
/// \param[in] schema the schema of the record batches to be written
-/// \param[in] options options for serialization
+/// \param[in] options options for serialization, optional
+/// \param[in] metadata custom metadata for File Footer, optional
/// \return Status
ARROW_EXPORT
Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
- const IpcWriteOptions& options = IpcWriteOptions::Defaults());
+ const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
+ const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
/// \brief Low-level API for writing a record batch (without schema)
/// to an OutputStream as encapsulated IPC message. See Arrow format