You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/05/08 15:51:23 UTC
[arrow] branch master updated: PARQUET-1571: [C++] Fix
BufferedInputStream when buffer exactly exhausted
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f15b213 PARQUET-1571: [C++] Fix BufferedInputStream when buffer exactly exhausted
f15b213 is described below
commit f15b2133e279f2153756b1b58d2ba8808aaa91c5
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Wed May 8 10:51:12 2019 -0500
PARQUET-1571: [C++] Fix BufferedInputStream when buffer exactly exhausted
Rework the BufferedInputStream implementation to make internal semantics more apparent.
Author: Antoine Pitrou <an...@python.org>
Closes #4275 from pitrou/PARQUET-1571-buffered-input-stream-fixes and squashes the following commits:
2a641edc7 <Antoine Pitrou> Remove setuptools_scm workaround
49914ef47 <Antoine Pitrou> PARQUET-1571: Fix BufferedInputStream when buffer exactly exhausted
---
cpp/src/parquet/util/memory-test.cc | 143 +++++++++++++++++++++++++++++-------
cpp/src/parquet/util/memory.cc | 52 ++++++++-----
cpp/src/parquet/util/memory.h | 5 +-
3 files changed, 155 insertions(+), 45 deletions(-)
diff --git a/cpp/src/parquet/util/memory-test.cc b/cpp/src/parquet/util/memory-test.cc
index 91671e2..8e33817 100644
--- a/cpp/src/parquet/util/memory-test.cc
+++ b/cpp/src/parquet/util/memory-test.cc
@@ -32,69 +32,158 @@ using arrow::MemoryPool;
namespace parquet {
-class TestBuffer : public ::testing::Test {};
-
-TEST(TestBufferedInputStream, Basics) {
- int64_t source_size = 256;
- int64_t stream_offset = 10;
- int64_t stream_size = source_size - stream_offset;
- int64_t chunk_size = 50;
- std::shared_ptr<ResizableBuffer> buf =
- AllocateBuffer(default_memory_pool(), source_size);
- ASSERT_EQ(source_size, buf->size());
- for (int i = 0; i < source_size; i++) {
- buf->mutable_data()[i] = static_cast<uint8_t>(i);
+class TestBufferedInputStream : public ::testing::Test {
+ public:
+ void SetUp() {
+ // Create a buffer larger than source size, to check that the stream end is upholded.
+ std::shared_ptr<ResizableBuffer> buf =
+ AllocateBuffer(default_memory_pool(), source_size_ + 10);
+ ASSERT_LT(source_size_, buf->size());
+ for (int i = 0; i < source_size_; i++) {
+ buf->mutable_data()[i] = static_cast<uint8_t>(i);
+ }
+ source_ = std::make_shared<ArrowInputFile>(
+ std::make_shared<::arrow::io::BufferReader>(buf));
+ stream_.reset(new BufferedInputStream(default_memory_pool(), chunk_size_,
+ source_.get(), stream_offset_, stream_size_));
}
- auto wrapper =
- std::make_shared<ArrowInputFile>(std::make_shared<::arrow::io::BufferReader>(buf));
-
- std::unique_ptr<BufferedInputStream> stream(new BufferedInputStream(
- default_memory_pool(), chunk_size, wrapper.get(), stream_offset, stream_size));
+ protected:
+ int64_t source_size_ = 256;
+ int64_t stream_offset_ = 10;
+ int64_t stream_size_ = source_size_ - stream_offset_;
+ int64_t chunk_size_ = 50;
+ std::shared_ptr<RandomAccessSource> source_;
+ std::unique_ptr<BufferedInputStream> stream_;
+};
+TEST_F(TestBufferedInputStream, Basics) {
const uint8_t* output;
int64_t bytes_read;
// source is at offset 10
- output = stream->Peek(10, &bytes_read);
+ output = stream_->Peek(10, &bytes_read);
ASSERT_EQ(10, bytes_read);
for (int i = 0; i < 10; i++) {
ASSERT_EQ(10 + i, output[i]) << i;
}
- output = stream->Read(10, &bytes_read);
+ output = stream_->Read(10, &bytes_read);
ASSERT_EQ(10, bytes_read);
for (int i = 0; i < 10; i++) {
ASSERT_EQ(10 + i, output[i]) << i;
}
- output = stream->Read(10, &bytes_read);
+ output = stream_->Read(10, &bytes_read);
ASSERT_EQ(10, bytes_read);
for (int i = 0; i < 10; i++) {
ASSERT_EQ(20 + i, output[i]) << i;
}
- stream->Advance(5);
- stream->Advance(5);
+ stream_->Advance(5);
+ stream_->Advance(5);
// source is at offset 40
// read across buffer boundary. buffer size is 50
- output = stream->Read(20, &bytes_read);
+ output = stream_->Read(20, &bytes_read);
ASSERT_EQ(20, bytes_read);
for (int i = 0; i < 20; i++) {
ASSERT_EQ(40 + i, output[i]) << i;
}
- // read more than original chunk_size
- output = stream->Read(60, &bytes_read);
+ // read more than original chunk size
+ output = stream_->Read(60, &bytes_read);
ASSERT_EQ(60, bytes_read);
for (int i = 0; i < 60; i++) {
ASSERT_EQ(60 + i, output[i]) << i;
}
- stream->Advance(120);
+ stream_->Advance(120);
// source is at offset 240
// read outside of source boundary. source size is 256
- output = stream->Read(30, &bytes_read);
+ output = stream_->Read(30, &bytes_read);
ASSERT_EQ(16, bytes_read);
for (int i = 0; i < 16; i++) {
ASSERT_EQ(240 + i, output[i]) << i;
}
+ // Stream exhausted
+ output = stream_->Read(1, &bytes_read);
+ ASSERT_EQ(bytes_read, 0);
+}
+
+TEST_F(TestBufferedInputStream, LargeFirstPeek) {
+ // Test a first peek larger than chunk size
+ const uint8_t* output;
+ int64_t bytes_read;
+ int64_t n = 70;
+ ASSERT_GT(n, chunk_size_);
+
+ // source is at offset 10
+ output = stream_->Peek(n, &bytes_read);
+ ASSERT_EQ(n, bytes_read);
+ for (int i = 0; i < n; i++) {
+ ASSERT_EQ(10 + i, output[i]) << i;
+ }
+ output = stream_->Peek(n, &bytes_read);
+ ASSERT_EQ(n, bytes_read);
+ for (int i = 0; i < n; i++) {
+ ASSERT_EQ(10 + i, output[i]) << i;
+ }
+ output = stream_->Read(n, &bytes_read);
+ ASSERT_EQ(n, bytes_read);
+ for (int i = 0; i < n; i++) {
+ ASSERT_EQ(10 + i, output[i]) << i;
+ }
+ // source is at offset 10 + n
+ output = stream_->Read(20, &bytes_read);
+ ASSERT_EQ(20, bytes_read);
+ for (int i = 0; i < 20; i++) {
+ ASSERT_EQ(10 + n + i, output[i]) << i;
+ }
+}
+
+TEST_F(TestBufferedInputStream, OneByteReads) {
+ const uint8_t* output;
+ int64_t bytes_read;
+
+ for (int i = 0; i < stream_size_; ++i) {
+ output = stream_->Read(1, &bytes_read);
+ ASSERT_EQ(bytes_read, 1);
+ ASSERT_EQ(10 + i, output[0]) << i;
+ }
+ // Stream exhausted
+ output = stream_->Read(1, &bytes_read);
+ ASSERT_EQ(bytes_read, 0);
+}
+
+TEST_F(TestBufferedInputStream, BufferExactlyExhausted) {
+ // Test exhausting the buffer exactly then issuing further reads (PARQUET-1571).
+ const uint8_t* output;
+ int64_t bytes_read;
+
+ // source is at offset 10
+ int64_t n = 10;
+ output = stream_->Read(n, &bytes_read);
+ ASSERT_EQ(n, bytes_read);
+ for (int i = 0; i < n; i++) {
+ ASSERT_EQ(10 + i, output[i]) << i;
+ }
+ // source is at offset 20
+ // Exhaust buffer exactly
+ n = stream_->remaining_in_buffer();
+ output = stream_->Read(n, &bytes_read);
+ ASSERT_EQ(n, bytes_read);
+ for (int i = 0; i < n; i++) {
+ ASSERT_EQ(20 + i, output[i]) << i;
+ }
+ // source is at offset 20 + n
+ // Read new buffer
+ output = stream_->Read(10, &bytes_read);
+ ASSERT_EQ(10, bytes_read);
+ for (int i = 0; i < 10; i++) {
+ ASSERT_EQ(20 + n + i, output[i]) << i;
+ }
+ // source is at offset 30 + n
+ output = stream_->Read(10, &bytes_read);
+ ASSERT_EQ(10, bytes_read);
+ for (int i = 0; i < 10; i++) {
+ ASSERT_EQ(30 + n + i, output[i]) << i;
+ }
}
TEST(TestArrowInputFile, ReadAt) {
diff --git a/cpp/src/parquet/util/memory.cc b/cpp/src/parquet/util/memory.cc
index e37fd88..9289601 100644
--- a/cpp/src/parquet/util/memory.cc
+++ b/cpp/src/parquet/util/memory.cc
@@ -256,42 +256,60 @@ BufferedInputStream::BufferedInputStream(MemoryPool* pool, int64_t buffer_size,
int64_t num_bytes)
: source_(source), stream_offset_(start), stream_end_(start + num_bytes) {
buffer_ = AllocateBuffer(pool, buffer_size);
- buffer_size_ = buffer_->size();
- // Required to force a lazy read
- buffer_offset_ = buffer_size_;
+ buffer_offset_ = 0;
+ buffer_end_ = 0;
+}
+
+int64_t BufferedInputStream::remaining_in_buffer() const {
+ return buffer_end_ - buffer_offset_;
}
const uint8_t* BufferedInputStream::Peek(int64_t num_to_peek, int64_t* num_bytes) {
- *num_bytes = std::min(num_to_peek, stream_end_ - stream_offset_);
- // increase the buffer size if needed
- if (*num_bytes > buffer_size_) {
- PARQUET_THROW_NOT_OK(buffer_->Resize(*num_bytes));
- buffer_size_ = buffer_->size();
- DCHECK(buffer_size_ >= *num_bytes);
+ int64_t buffer_avail = buffer_end_ - buffer_offset_;
+ int64_t stream_avail = stream_end_ - stream_offset_;
+ // Do not try to peek more than the total remaining number of bytes.
+ *num_bytes = std::min(num_to_peek, buffer_avail + stream_avail);
+ // Increase the buffer size if needed
+ if (*num_bytes > buffer_->size() - buffer_offset_) {
+ // XXX Should adopt a shrinking heuristic if buffer_offset_ is close
+ // to buffer_end_.
+ PARQUET_THROW_NOT_OK(buffer_->Resize(*num_bytes + buffer_offset_));
+ DCHECK(buffer_->size() - buffer_offset_ >= *num_bytes);
}
- // Read more data when buffer has insufficient left or when resized
- if (*num_bytes > (buffer_size_ - buffer_offset_)) {
- buffer_size_ = std::min(buffer_size_, stream_end_ - stream_offset_);
+ // Read more data when buffer has insufficient left
+ if (*num_bytes > buffer_avail) {
+ // Read as much as possible to fill the buffer, but not past stream end
+ int64_t read_size = std::min(buffer_->size() - buffer_end_, stream_avail);
int64_t bytes_read =
- source_->ReadAt(stream_offset_, buffer_size_, buffer_->mutable_data());
- if (bytes_read < *num_bytes) {
+ source_->ReadAt(stream_offset_, read_size, buffer_->mutable_data() + buffer_end_);
+ stream_offset_ += bytes_read;
+ buffer_end_ += bytes_read;
+ if (bytes_read < read_size) {
throw ParquetException("Failed reading column data from source");
}
- buffer_offset_ = 0;
}
+ DCHECK(*num_bytes <= buffer_end_ - buffer_offset_); // Enough bytes available
return buffer_->data() + buffer_offset_;
}
const uint8_t* BufferedInputStream::Read(int64_t num_to_read, int64_t* num_bytes) {
const uint8_t* result = Peek(num_to_read, num_bytes);
- stream_offset_ += *num_bytes;
buffer_offset_ += *num_bytes;
+ if (buffer_offset_ == buffer_end_) {
+ // Rewind pointer to reuse beginning of buffer
+ buffer_offset_ = 0;
+ buffer_end_ = 0;
+ }
return result;
}
void BufferedInputStream::Advance(int64_t num_bytes) {
- stream_offset_ += num_bytes;
buffer_offset_ += num_bytes;
+ if (buffer_offset_ == buffer_end_) {
+ // Rewind pointer to reuse beginning of buffer
+ buffer_offset_ = 0;
+ buffer_end_ = 0;
+ }
}
std::shared_ptr<ResizableBuffer> AllocateBuffer(MemoryPool* pool, int64_t size) {
diff --git a/cpp/src/parquet/util/memory.h b/cpp/src/parquet/util/memory.h
index 02cf082..0d23c31 100644
--- a/cpp/src/parquet/util/memory.h
+++ b/cpp/src/parquet/util/memory.h
@@ -276,13 +276,16 @@ class PARQUET_EXPORT BufferedInputStream : public InputStream {
virtual void Advance(int64_t num_bytes);
+ // Return the number of bytes remaining in buffer (i.e. without reading from source).
+ int64_t remaining_in_buffer() const;
+
private:
std::shared_ptr<ResizableBuffer> buffer_;
RandomAccessSource* source_;
int64_t stream_offset_;
int64_t stream_end_;
int64_t buffer_offset_;
- int64_t buffer_size_;
+ int64_t buffer_end_;
};
std::shared_ptr<ResizableBuffer> PARQUET_EXPORT AllocateBuffer(