You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2020/04/04 23:02:52 UTC

[arrow] branch master updated: ARROW-6837: [C++] Add APIs to read and write "custom_metadata" field of IPC file footer

This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new 9f99a98  ARROW-6837: [C++] Add APIs to read and write "custom_metadata" field of IPC file footer
9f99a98 is described below

commit 9f99a989853694b20d85130d53f865288206ffc3
Author: John Muehlhausen <jm...@kershnertrading.com>
AuthorDate: Sat Apr 4 18:02:36 2020 -0500

    ARROW-6837: [C++] Add APIs to read and write "custom_metadata" field of IPC file footer
    
    Closes #6838 from wesm/ARROW-6837
    
    Lead-authored-by: John Muehlhausen <jm...@kershnertrading.com>
    Co-authored-by: Wes McKinney <we...@apache.org>
    Signed-off-by: Wes McKinney <we...@apache.org>
---
 cpp/src/arrow/ipc/metadata_internal.cc |  8 ++++++--
 cpp/src/arrow/ipc/metadata_internal.h  |  1 +
 cpp/src/arrow/ipc/read_write_test.cc   | 29 +++++++++++++++++++++++++++--
 cpp/src/arrow/ipc/reader.cc            |  8 ++++++++
 cpp/src/arrow/ipc/reader.h             |  4 ++++
 cpp/src/arrow/ipc/writer.cc            | 13 +++++++++----
 cpp/src/arrow/ipc/writer.h             |  6 ++++--
 7 files changed, 59 insertions(+), 10 deletions(-)

diff --git a/cpp/src/arrow/ipc/metadata_internal.cc b/cpp/src/arrow/ipc/metadata_internal.cc
index 365a6cc..8287b06 100644
--- a/cpp/src/arrow/ipc/metadata_internal.cc
+++ b/cpp/src/arrow/ipc/metadata_internal.cc
@@ -1184,6 +1184,7 @@ FileBlocksToFlatbuffer(FBB& fbb, const std::vector<FileBlock>& blocks) {
 
 Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
                        const std::vector<FileBlock>& record_batches,
+                       const std::shared_ptr<const KeyValueMetadata>& metadata,
                        io::OutputStream* out) {
   FBB fbb;
 
@@ -1208,8 +1209,11 @@ Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dicti
   auto fb_dictionaries = FileBlocksToFlatbuffer(fbb, dictionaries);
   auto fb_record_batches = FileBlocksToFlatbuffer(fbb, record_batches);
 
-  auto footer = flatbuf::CreateFooter(fbb, kCurrentMetadataVersion, fb_schema,
-                                      fb_dictionaries, fb_record_batches);
+  auto fb_custom_metadata = SerializeCustomMetadata(fbb, metadata);
+
+  auto footer =
+      flatbuf::CreateFooter(fbb, kCurrentMetadataVersion, fb_schema, fb_dictionaries,
+                            fb_record_batches, fb_custom_metadata);
   fbb.Finish(footer);
 
   int32_t size = fbb.GetSize();
diff --git a/cpp/src/arrow/ipc/metadata_internal.h b/cpp/src/arrow/ipc/metadata_internal.h
index 8706fee..617f9f4 100644
--- a/cpp/src/arrow/ipc/metadata_internal.h
+++ b/cpp/src/arrow/ipc/metadata_internal.h
@@ -190,6 +190,7 @@ Result<std::shared_ptr<Buffer>> WriteSparseTensorMessage(
 
 Status WriteFileFooter(const Schema& schema, const std::vector<FileBlock>& dictionaries,
                        const std::vector<FileBlock>& record_batches,
+                       const std::shared_ptr<const KeyValueMetadata>& metadata,
                        io::OutputStream* out);
 
 Status WriteDictionaryMessage(
diff --git a/cpp/src/arrow/ipc/read_write_test.cc b/cpp/src/arrow/ipc/read_write_test.cc
index 84b834f..d1e39d8 100644
--- a/cpp/src/arrow/ipc/read_write_test.cc
+++ b/cpp/src/arrow/ipc/read_write_test.cc
@@ -789,12 +789,13 @@ TEST_F(RecursionLimits, StressLimit) {
 #endif  // !defined(_WIN32) || defined(NDEBUG)
 
 struct FileWriterHelper {
-  Status Init(const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options) {
+  Status Init(const std::shared_ptr<Schema>& schema, const IpcWriteOptions& options,
+              const std::shared_ptr<const KeyValueMetadata>& metadata = nullptr) {
     num_batches_written_ = 0;
 
     RETURN_NOT_OK(AllocateResizableBuffer(0, &buffer_));
     sink_.reset(new io::BufferOutputStream(buffer_));
-    ARROW_ASSIGN_OR_RAISE(writer_, NewFileWriter(sink_.get(), schema, options));
+    ARROW_ASSIGN_OR_RAISE(writer_, NewFileWriter(sink_.get(), schema, options, metadata));
     return Status::OK();
   }
 
@@ -836,6 +837,13 @@ struct FileWriterHelper {
     return Status::OK();
   }
 
+  Result<std::shared_ptr<const KeyValueMetadata>> ReadFooterMetadata() {
+    auto buf_reader = std::make_shared<io::BufferReader>(buffer_);
+    ARROW_ASSIGN_OR_RAISE(auto reader,
+                          RecordBatchFileReader::Open(buf_reader.get(), footer_offset_));
+    return reader->metadata();
+  }
+
   std::shared_ptr<ResizableBuffer> buffer_;
   std::unique_ptr<io::BufferOutputStream> sink_;
   std::shared_ptr<RecordBatchWriter> writer_;
@@ -1090,6 +1098,23 @@ INSTANTIATE_TEST_SUITE_P(GenericIpcRoundTripTests, TestIpcRoundTrip, BATCH_CASES
 INSTANTIATE_TEST_SUITE_P(FileRoundTripTests, TestFileFormat, BATCH_CASES());
 INSTANTIATE_TEST_SUITE_P(StreamRoundTripTests, TestStreamFormat, BATCH_CASES());
 
+TEST(TestIpcFileFormat, FooterMetaData) {
+  // ARROW-6837
+  std::shared_ptr<RecordBatch> batch;
+  ASSERT_OK(MakeIntRecordBatch(&batch));
+
+  auto metadata = key_value_metadata({"ARROW:example", "ARROW:example2"},
+                                     {"something something", "something something2"});
+
+  FileWriterHelper helper;
+  ASSERT_OK(helper.Init(batch->schema(), IpcWriteOptions::Defaults(), metadata));
+  ASSERT_OK(helper.WriteBatch(batch));
+  ASSERT_OK(helper.Finish());
+
+  ASSERT_OK_AND_ASSIGN(auto out_metadata, helper.ReadFooterMetadata());
+  ASSERT_TRUE(out_metadata->Equals(*metadata));
+}
+
 // This test uses uninitialized memory
 
 #if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index 8105987..9cefb51 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -799,6 +799,8 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
 
   std::shared_ptr<Schema> schema() const override { return schema_; }
 
+  std::shared_ptr<const KeyValueMetadata> metadata() const override { return metadata_; }
+
  private:
   FileBlock GetRecordBatchBlock(int i) const {
     return FileBlockFromFlatbuffer(footer_->recordBatches()->Get(i));
@@ -874,6 +876,11 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
     }
     footer_ = flatbuf::GetFooter(data);
 
+    auto fb_metadata = footer_->custom_metadata();
+    if (fb_metadata != nullptr) {
+      RETURN_NOT_OK(internal::GetKeyValueMetadata(fb_metadata, &metadata_));
+    }
+
     return Status::OK();
   }
 
@@ -894,6 +901,7 @@ class RecordBatchFileReaderImpl : public RecordBatchFileReader {
   // Footer metadata
   std::shared_ptr<Buffer> footer_buffer_;
   const flatbuf::Footer* footer_;
+  std::shared_ptr<const KeyValueMetadata> metadata_;
 
   bool read_dictionaries_ = false;
   DictionaryMemo dictionary_memo_;
diff --git a/cpp/src/arrow/ipc/reader.h b/cpp/src/arrow/ipc/reader.h
index dd2ebd4..903b952 100644
--- a/cpp/src/arrow/ipc/reader.h
+++ b/cpp/src/arrow/ipc/reader.h
@@ -177,6 +177,10 @@ class ARROW_EXPORT RecordBatchFileReader {
   /// \brief Return the metadata version from the file metadata
   virtual MetadataVersion version() const = 0;
 
+  /// \brief Return the contents of the custom_metadata field from the file's
+  /// Footer
+  virtual std::shared_ptr<const KeyValueMetadata> metadata() const = 0;
+
   /// \brief Read a particular record batch from the file. Does not copy memory
   /// if the input source supports zero-copy.
   ///
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index ccec08a..ad27fb8 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -1102,8 +1102,9 @@ class PayloadStreamWriter : public IpcPayloadWriter, protected StreamBookKeeper
 class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBookKeeper {
  public:
   PayloadFileWriter(const IpcWriteOptions& options, const std::shared_ptr<Schema>& schema,
+                    const std::shared_ptr<const KeyValueMetadata>& metadata,
                     io::OutputStream* sink)
-      : StreamBookKeeper(options, sink), schema_(schema) {}
+      : StreamBookKeeper(options, sink), schema_(schema), metadata_(metadata) {}
 
   ~PayloadFileWriter() override = default;
 
@@ -1153,7 +1154,8 @@ class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBoo
     // Write file footer
     RETURN_NOT_OK(UpdatePosition());
     int64_t initial_position = position_;
-    RETURN_NOT_OK(WriteFileFooter(*schema_, dictionaries_, record_batches_, sink_));
+    RETURN_NOT_OK(
+        WriteFileFooter(*schema_, dictionaries_, record_batches_, metadata_, sink_));
 
     // Write footer length
     RETURN_NOT_OK(UpdatePosition());
@@ -1170,6 +1172,7 @@ class PayloadFileWriter : public internal::IpcPayloadWriter, protected StreamBoo
 
  protected:
   std::shared_ptr<Schema> schema_;
+  std::shared_ptr<const KeyValueMetadata> metadata_;
   std::vector<FileBlock> dictionaries_;
   std::vector<FileBlock> record_batches_;
 };
@@ -1186,9 +1189,11 @@ Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
 
 Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
     io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    const IpcWriteOptions& options) {
+    const IpcWriteOptions& options,
+    const std::shared_ptr<const KeyValueMetadata>& metadata) {
   return std::make_shared<internal::IpcFormatWriter>(
-      ::arrow::internal::make_unique<internal::PayloadFileWriter>(options, schema, sink),
+      ::arrow::internal::make_unique<internal::PayloadFileWriter>(options, schema,
+                                                                  metadata, sink),
       schema, options);
 }
 
diff --git a/cpp/src/arrow/ipc/writer.h b/cpp/src/arrow/ipc/writer.h
index 2d47a37..f2b057e 100644
--- a/cpp/src/arrow/ipc/writer.h
+++ b/cpp/src/arrow/ipc/writer.h
@@ -99,12 +99,14 @@ Result<std::shared_ptr<RecordBatchWriter>> NewStreamWriter(
 ///
 /// \param[in] sink output stream to write to
 /// \param[in] schema the schema of the record batches to be written
-/// \param[in] options options for serialization
+/// \param[in] options options for serialization, optional
+/// \param[in] metadata custom metadata for File Footer, optional
 /// \return Status
 ARROW_EXPORT
 Result<std::shared_ptr<RecordBatchWriter>> NewFileWriter(
     io::OutputStream* sink, const std::shared_ptr<Schema>& schema,
-    const IpcWriteOptions& options = IpcWriteOptions::Defaults());
+    const IpcWriteOptions& options = IpcWriteOptions::Defaults(),
+    const std::shared_ptr<const KeyValueMetadata>& metadata = NULLPTR);
 
 /// \brief Low-level API for writing a record batch (without schema)
 /// to an OutputStream as encapsulated IPC message. See Arrow format