You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/05/08 15:51:23 UTC

[arrow] branch master updated: PARQUET-1571: [C++] Fix BufferedInputStream when buffer exactly exhausted

This is an automated email from the ASF dual-hosted git repository.

wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/master by this push:
     new f15b213  PARQUET-1571: [C++] Fix BufferedInputStream when buffer exactly exhausted
f15b213 is described below

commit f15b2133e279f2153756b1b58d2ba8808aaa91c5
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Wed May 8 10:51:12 2019 -0500

    PARQUET-1571: [C++] Fix BufferedInputStream when buffer exactly exhausted
    
    Rework the BufferedInputStream implementation to make internal semantics more apparent.
    
    Author: Antoine Pitrou <an...@python.org>
    
    Closes #4275 from pitrou/PARQUET-1571-buffered-input-stream-fixes and squashes the following commits:
    
    2a641edc7 <Antoine Pitrou> Remove setuptools_scm workaround
    49914ef47 <Antoine Pitrou> PARQUET-1571:  Fix BufferedInputStream when buffer exactly exhausted
---
 cpp/src/parquet/util/memory-test.cc | 143 +++++++++++++++++++++++++++++-------
 cpp/src/parquet/util/memory.cc      |  52 ++++++++-----
 cpp/src/parquet/util/memory.h       |   5 +-
 3 files changed, 155 insertions(+), 45 deletions(-)

diff --git a/cpp/src/parquet/util/memory-test.cc b/cpp/src/parquet/util/memory-test.cc
index 91671e2..8e33817 100644
--- a/cpp/src/parquet/util/memory-test.cc
+++ b/cpp/src/parquet/util/memory-test.cc
@@ -32,69 +32,158 @@ using arrow::MemoryPool;
 
 namespace parquet {
 
-class TestBuffer : public ::testing::Test {};
-
-TEST(TestBufferedInputStream, Basics) {
-  int64_t source_size = 256;
-  int64_t stream_offset = 10;
-  int64_t stream_size = source_size - stream_offset;
-  int64_t chunk_size = 50;
-  std::shared_ptr<ResizableBuffer> buf =
-      AllocateBuffer(default_memory_pool(), source_size);
-  ASSERT_EQ(source_size, buf->size());
-  for (int i = 0; i < source_size; i++) {
-    buf->mutable_data()[i] = static_cast<uint8_t>(i);
+class TestBufferedInputStream : public ::testing::Test {
+ public:
+  void SetUp() {
+    // Create a buffer larger than source size, to check that the stream end is upholded.
+    std::shared_ptr<ResizableBuffer> buf =
+        AllocateBuffer(default_memory_pool(), source_size_ + 10);
+    ASSERT_LT(source_size_, buf->size());
+    for (int i = 0; i < source_size_; i++) {
+      buf->mutable_data()[i] = static_cast<uint8_t>(i);
+    }
+    source_ = std::make_shared<ArrowInputFile>(
+        std::make_shared<::arrow::io::BufferReader>(buf));
+    stream_.reset(new BufferedInputStream(default_memory_pool(), chunk_size_,
+                                          source_.get(), stream_offset_, stream_size_));
   }
 
-  auto wrapper =
-      std::make_shared<ArrowInputFile>(std::make_shared<::arrow::io::BufferReader>(buf));
-
-  std::unique_ptr<BufferedInputStream> stream(new BufferedInputStream(
-      default_memory_pool(), chunk_size, wrapper.get(), stream_offset, stream_size));
+ protected:
+  int64_t source_size_ = 256;
+  int64_t stream_offset_ = 10;
+  int64_t stream_size_ = source_size_ - stream_offset_;
+  int64_t chunk_size_ = 50;
+  std::shared_ptr<RandomAccessSource> source_;
+  std::unique_ptr<BufferedInputStream> stream_;
+};
 
+TEST_F(TestBufferedInputStream, Basics) {
   const uint8_t* output;
   int64_t bytes_read;
 
   // source is at offset 10
-  output = stream->Peek(10, &bytes_read);
+  output = stream_->Peek(10, &bytes_read);
   ASSERT_EQ(10, bytes_read);
   for (int i = 0; i < 10; i++) {
     ASSERT_EQ(10 + i, output[i]) << i;
   }
-  output = stream->Read(10, &bytes_read);
+  output = stream_->Read(10, &bytes_read);
   ASSERT_EQ(10, bytes_read);
   for (int i = 0; i < 10; i++) {
     ASSERT_EQ(10 + i, output[i]) << i;
   }
-  output = stream->Read(10, &bytes_read);
+  output = stream_->Read(10, &bytes_read);
   ASSERT_EQ(10, bytes_read);
   for (int i = 0; i < 10; i++) {
     ASSERT_EQ(20 + i, output[i]) << i;
   }
-  stream->Advance(5);
-  stream->Advance(5);
+  stream_->Advance(5);
+  stream_->Advance(5);
   // source is at offset 40
   // read across buffer boundary. buffer size is 50
-  output = stream->Read(20, &bytes_read);
+  output = stream_->Read(20, &bytes_read);
   ASSERT_EQ(20, bytes_read);
   for (int i = 0; i < 20; i++) {
     ASSERT_EQ(40 + i, output[i]) << i;
   }
-  // read more than original chunk_size
-  output = stream->Read(60, &bytes_read);
+  // read more than original chunk size
+  output = stream_->Read(60, &bytes_read);
   ASSERT_EQ(60, bytes_read);
   for (int i = 0; i < 60; i++) {
     ASSERT_EQ(60 + i, output[i]) << i;
   }
 
-  stream->Advance(120);
+  stream_->Advance(120);
   // source is at offset 240
   // read outside of source boundary. source size is 256
-  output = stream->Read(30, &bytes_read);
+  output = stream_->Read(30, &bytes_read);
   ASSERT_EQ(16, bytes_read);
   for (int i = 0; i < 16; i++) {
     ASSERT_EQ(240 + i, output[i]) << i;
   }
+  // Stream exhausted
+  output = stream_->Read(1, &bytes_read);
+  ASSERT_EQ(bytes_read, 0);
+}
+
+TEST_F(TestBufferedInputStream, LargeFirstPeek) {
+  // Test a first peek larger than chunk size
+  const uint8_t* output;
+  int64_t bytes_read;
+  int64_t n = 70;
+  ASSERT_GT(n, chunk_size_);
+
+  // source is at offset 10
+  output = stream_->Peek(n, &bytes_read);
+  ASSERT_EQ(n, bytes_read);
+  for (int i = 0; i < n; i++) {
+    ASSERT_EQ(10 + i, output[i]) << i;
+  }
+  output = stream_->Peek(n, &bytes_read);
+  ASSERT_EQ(n, bytes_read);
+  for (int i = 0; i < n; i++) {
+    ASSERT_EQ(10 + i, output[i]) << i;
+  }
+  output = stream_->Read(n, &bytes_read);
+  ASSERT_EQ(n, bytes_read);
+  for (int i = 0; i < n; i++) {
+    ASSERT_EQ(10 + i, output[i]) << i;
+  }
+  // source is at offset 10 + n
+  output = stream_->Read(20, &bytes_read);
+  ASSERT_EQ(20, bytes_read);
+  for (int i = 0; i < 20; i++) {
+    ASSERT_EQ(10 + n + i, output[i]) << i;
+  }
+}
+
+TEST_F(TestBufferedInputStream, OneByteReads) {
+  const uint8_t* output;
+  int64_t bytes_read;
+
+  for (int i = 0; i < stream_size_; ++i) {
+    output = stream_->Read(1, &bytes_read);
+    ASSERT_EQ(bytes_read, 1);
+    ASSERT_EQ(10 + i, output[0]) << i;
+  }
+  // Stream exhausted
+  output = stream_->Read(1, &bytes_read);
+  ASSERT_EQ(bytes_read, 0);
+}
+
+TEST_F(TestBufferedInputStream, BufferExactlyExhausted) {
+  // Test exhausting the buffer exactly then issuing further reads (PARQUET-1571).
+  const uint8_t* output;
+  int64_t bytes_read;
+
+  // source is at offset 10
+  int64_t n = 10;
+  output = stream_->Read(n, &bytes_read);
+  ASSERT_EQ(n, bytes_read);
+  for (int i = 0; i < n; i++) {
+    ASSERT_EQ(10 + i, output[i]) << i;
+  }
+  // source is at offset 20
+  // Exhaust buffer exactly
+  n = stream_->remaining_in_buffer();
+  output = stream_->Read(n, &bytes_read);
+  ASSERT_EQ(n, bytes_read);
+  for (int i = 0; i < n; i++) {
+    ASSERT_EQ(20 + i, output[i]) << i;
+  }
+  // source is at offset 20 + n
+  // Read new buffer
+  output = stream_->Read(10, &bytes_read);
+  ASSERT_EQ(10, bytes_read);
+  for (int i = 0; i < 10; i++) {
+    ASSERT_EQ(20 + n + i, output[i]) << i;
+  }
+  // source is at offset 30 + n
+  output = stream_->Read(10, &bytes_read);
+  ASSERT_EQ(10, bytes_read);
+  for (int i = 0; i < 10; i++) {
+    ASSERT_EQ(30 + n + i, output[i]) << i;
+  }
 }
 
 TEST(TestArrowInputFile, ReadAt) {
diff --git a/cpp/src/parquet/util/memory.cc b/cpp/src/parquet/util/memory.cc
index e37fd88..9289601 100644
--- a/cpp/src/parquet/util/memory.cc
+++ b/cpp/src/parquet/util/memory.cc
@@ -256,42 +256,60 @@ BufferedInputStream::BufferedInputStream(MemoryPool* pool, int64_t buffer_size,
                                          int64_t num_bytes)
     : source_(source), stream_offset_(start), stream_end_(start + num_bytes) {
   buffer_ = AllocateBuffer(pool, buffer_size);
-  buffer_size_ = buffer_->size();
-  // Required to force a lazy read
-  buffer_offset_ = buffer_size_;
+  buffer_offset_ = 0;
+  buffer_end_ = 0;
+}
+
+int64_t BufferedInputStream::remaining_in_buffer() const {
+  return buffer_end_ - buffer_offset_;
 }
 
 const uint8_t* BufferedInputStream::Peek(int64_t num_to_peek, int64_t* num_bytes) {
-  *num_bytes = std::min(num_to_peek, stream_end_ - stream_offset_);
-  // increase the buffer size if needed
-  if (*num_bytes > buffer_size_) {
-    PARQUET_THROW_NOT_OK(buffer_->Resize(*num_bytes));
-    buffer_size_ = buffer_->size();
-    DCHECK(buffer_size_ >= *num_bytes);
+  int64_t buffer_avail = buffer_end_ - buffer_offset_;
+  int64_t stream_avail = stream_end_ - stream_offset_;
+  // Do not try to peek more than the total remaining number of bytes.
+  *num_bytes = std::min(num_to_peek, buffer_avail + stream_avail);
+  // Increase the buffer size if needed
+  if (*num_bytes > buffer_->size() - buffer_offset_) {
+    // XXX Should adopt a shrinking heuristic if buffer_offset_ is close
+    // to buffer_end_.
+    PARQUET_THROW_NOT_OK(buffer_->Resize(*num_bytes + buffer_offset_));
+    DCHECK(buffer_->size() - buffer_offset_ >= *num_bytes);
   }
-  // Read more data when buffer has insufficient left or when resized
-  if (*num_bytes > (buffer_size_ - buffer_offset_)) {
-    buffer_size_ = std::min(buffer_size_, stream_end_ - stream_offset_);
+  // Read more data when buffer has insufficient left
+  if (*num_bytes > buffer_avail) {
+    // Read as much as possible to fill the buffer, but not past stream end
+    int64_t read_size = std::min(buffer_->size() - buffer_end_, stream_avail);
     int64_t bytes_read =
-        source_->ReadAt(stream_offset_, buffer_size_, buffer_->mutable_data());
-    if (bytes_read < *num_bytes) {
+        source_->ReadAt(stream_offset_, read_size, buffer_->mutable_data() + buffer_end_);
+    stream_offset_ += bytes_read;
+    buffer_end_ += bytes_read;
+    if (bytes_read < read_size) {
       throw ParquetException("Failed reading column data from source");
     }
-    buffer_offset_ = 0;
   }
+  DCHECK(*num_bytes <= buffer_end_ - buffer_offset_);  // Enough bytes available
   return buffer_->data() + buffer_offset_;
 }
 
 const uint8_t* BufferedInputStream::Read(int64_t num_to_read, int64_t* num_bytes) {
   const uint8_t* result = Peek(num_to_read, num_bytes);
-  stream_offset_ += *num_bytes;
   buffer_offset_ += *num_bytes;
+  if (buffer_offset_ == buffer_end_) {
+    // Rewind pointer to reuse beginning of buffer
+    buffer_offset_ = 0;
+    buffer_end_ = 0;
+  }
   return result;
 }
 
 void BufferedInputStream::Advance(int64_t num_bytes) {
-  stream_offset_ += num_bytes;
   buffer_offset_ += num_bytes;
+  if (buffer_offset_ == buffer_end_) {
+    // Rewind pointer to reuse beginning of buffer
+    buffer_offset_ = 0;
+    buffer_end_ = 0;
+  }
 }
 
 std::shared_ptr<ResizableBuffer> AllocateBuffer(MemoryPool* pool, int64_t size) {
diff --git a/cpp/src/parquet/util/memory.h b/cpp/src/parquet/util/memory.h
index 02cf082..0d23c31 100644
--- a/cpp/src/parquet/util/memory.h
+++ b/cpp/src/parquet/util/memory.h
@@ -276,13 +276,16 @@ class PARQUET_EXPORT BufferedInputStream : public InputStream {
 
   virtual void Advance(int64_t num_bytes);
 
+  // Return the number of bytes remaining in buffer (i.e. without reading from source).
+  int64_t remaining_in_buffer() const;
+
  private:
   std::shared_ptr<ResizableBuffer> buffer_;
   RandomAccessSource* source_;
   int64_t stream_offset_;
   int64_t stream_end_;
   int64_t buffer_offset_;
-  int64_t buffer_size_;
+  int64_t buffer_end_;
 };
 
 std::shared_ptr<ResizableBuffer> PARQUET_EXPORT AllocateBuffer(