You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/12/06 00:06:07 UTC
[arrow] branch master updated: ARROW-3291: [C++] Add
string_view-based constructor for BufferReader
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 187b98e ARROW-3291: [C++] Add string_view-based constructor for BufferReader
187b98e is described below
commit 187b98ed338d4995317dae9efd19870c532192cb
Author: Wes McKinney <we...@apache.org>
AuthorDate: Wed Dec 5 18:05:58 2018 -0600
ARROW-3291: [C++] Add string_view-based constructor for BufferReader
This makes the API simpler for creating emphemeral IO sources. The caller is responsible for managing memory lifetime in this case
Author: Wes McKinney <we...@apache.org>
Closes #3093 from wesm/ARROW-3291 and squashes the following commits:
125d37546 <Wes McKinney> Add string_view-based constructor for BufferReader, use in various unit tests
---
cpp/src/arrow/flight/types.cc | 3 +--
cpp/src/arrow/io/io-memory-test.cc | 22 ++++++++++++++++++++--
cpp/src/arrow/io/io-readahead-test.cc | 5 ++---
cpp/src/arrow/io/memory.h | 7 +++++++
cpp/src/arrow/ipc/feather-test.cc | 4 ++--
cpp/src/parquet/util/memory-test.cc | 5 ++---
6 files changed, 34 insertions(+), 12 deletions(-)
diff --git a/cpp/src/arrow/flight/types.cc b/cpp/src/arrow/flight/types.cc
index 8c7588d..aba93ad 100644
--- a/cpp/src/arrow/flight/types.cc
+++ b/cpp/src/arrow/flight/types.cc
@@ -36,8 +36,7 @@ Status FlightInfo::GetSchema(std::shared_ptr<Schema>* out) const {
}
/// XXX(wesm): arrow::ipc::ReadSchema in its current form will not suffice
/// for reading schemas with dictionaries. See ARROW-3144
- io::BufferReader schema_reader(reinterpret_cast<const uint8_t*>(data_.schema.c_str()),
- static_cast<int64_t>(data_.schema.size()));
+ io::BufferReader schema_reader(data_.schema);
RETURN_NOT_OK(ipc::ReadSchema(&schema_reader, &schema_));
reconstructed_schema_ = true;
*out = schema_;
diff --git a/cpp/src/arrow/io/io-memory-test.cc b/cpp/src/arrow/io/io-memory-test.cc
index cb8a8b5..ecd920b 100644
--- a/cpp/src/arrow/io/io-memory-test.cc
+++ b/cpp/src/arrow/io/io-memory-test.cc
@@ -139,11 +139,29 @@ TEST(TestFixedSizeBufferWriter, Basics) {
ASSERT_OK(writer.Close());
}
+TEST(TestBufferReader, FromStrings) {
+ // ARROW-3291: construct BufferReader from std::string or
+ // arrow::util::string_view
+
+ std::string data = "data123456";
+ auto view = util::string_view(data);
+
+ BufferReader reader1(data);
+ BufferReader reader2(view);
+
+ std::shared_ptr<Buffer> piece;
+ ASSERT_OK(reader1.Read(4, &piece));
+ ASSERT_EQ(0, memcmp(piece->data(), data.data(), 4));
+
+ ASSERT_OK(reader2.Seek(2));
+ ASSERT_OK(reader2.Read(4, &piece));
+ ASSERT_EQ(0, memcmp(piece->data(), data.data() + 2, 4));
+}
+
TEST(TestBufferReader, Seeking) {
std::string data = "data123456";
- auto buffer = std::make_shared<Buffer>(data);
- BufferReader reader(buffer);
+ BufferReader reader(data);
int64_t pos;
ASSERT_OK(reader.Tell(&pos));
ASSERT_EQ(pos, 0);
diff --git a/cpp/src/arrow/io/io-readahead-test.cc b/cpp/src/arrow/io/io-readahead-test.cc
index 1e5d02a..b7f404f 100644
--- a/cpp/src/arrow/io/io-readahead-test.cc
+++ b/cpp/src/arrow/io/io-readahead-test.cc
@@ -59,9 +59,8 @@ static void busy_wait(double seconds, std::function<bool()> predicate) {
std::shared_ptr<BufferReader> DataReader(const std::string& data) {
std::shared_ptr<Buffer> buffer;
- ABORT_NOT_OK(AllocateBuffer(data.length(), &buffer));
- memcpy(buffer->mutable_data(), data.data(), data.length());
- return std::make_shared<BufferReader>(std::move(buffer));
+ ABORT_NOT_OK(Buffer::FromString(data, &buffer));
+ return std::make_shared<BufferReader>(buffer);
}
static int64_t WaitForPosition(const RandomAccessFile& file, int64_t expected,
diff --git a/cpp/src/arrow/io/memory.h b/cpp/src/arrow/io/memory.h
index 928ed10..cf73def 100644
--- a/cpp/src/arrow/io/memory.h
+++ b/cpp/src/arrow/io/memory.h
@@ -25,6 +25,7 @@
#include "arrow/io/interfaces.h"
#include "arrow/memory_pool.h"
+#include "arrow/util/string_view.h"
#include "arrow/util/visibility.h"
namespace arrow {
@@ -133,6 +134,12 @@ class ARROW_EXPORT BufferReader : public RandomAccessFile {
explicit BufferReader(const Buffer& buffer);
BufferReader(const uint8_t* data, int64_t size);
+ /// \brief Instantiate from std::string or arrow::util::string_view. Does not
+ /// own data
+ explicit BufferReader(const util::string_view& data)
+ : BufferReader(reinterpret_cast<const uint8_t*>(data.data()),
+ static_cast<int64_t>(data.size())) {}
+
Status Close() override;
bool closed() const override;
Status Tell(int64_t* position) const override;
diff --git a/cpp/src/arrow/ipc/feather-test.cc b/cpp/src/arrow/ipc/feather-test.cc
index d032710..b0be289 100644
--- a/cpp/src/arrow/ipc/feather-test.cc
+++ b/cpp/src/arrow/ipc/feather-test.cc
@@ -289,7 +289,7 @@ class TestTableReader : public ::testing::Test {
ASSERT_OK(stream_->Finish(&output_));
- std::shared_ptr<io::BufferReader> buffer(new io::BufferReader(output_));
+ auto buffer = std::make_shared<io::BufferReader>(output_);
ASSERT_OK(TableReader::Open(buffer, &reader_));
}
@@ -364,7 +364,7 @@ class TestTableWriter : public ::testing::Test {
ASSERT_OK(stream_->Finish(&output_));
- std::shared_ptr<io::BufferReader> buffer(new io::BufferReader(output_));
+ auto buffer = std::make_shared<io::BufferReader>(output_);
ASSERT_OK(TableReader::Open(buffer, &reader_));
}
diff --git a/cpp/src/parquet/util/memory-test.cc b/cpp/src/parquet/util/memory-test.cc
index 58903b6..cdeb8ee 100644
--- a/cpp/src/parquet/util/memory-test.cc
+++ b/cpp/src/parquet/util/memory-test.cc
@@ -99,9 +99,8 @@ TEST(TestBufferedInputStream, Basics) {
TEST(TestArrowInputFile, ReadAt) {
std::string data = "this is the data";
- auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
- auto file = std::make_shared<::arrow::io::BufferReader>(data_buffer, data.size());
+ auto file = std::make_shared<::arrow::io::BufferReader>(data);
auto source = std::make_shared<ArrowInputFile>(file);
ASSERT_EQ(0, source->Tell());
@@ -119,7 +118,7 @@ TEST(TestArrowInputFile, Read) {
std::string data = "this is the data";
auto data_buffer = reinterpret_cast<const uint8_t*>(data.c_str());
- auto file = std::make_shared<::arrow::io::BufferReader>(data_buffer, data.size());
+ auto file = std::make_shared<::arrow::io::BufferReader>(data);
auto source = std::make_shared<ArrowInputFile>(file);
ASSERT_EQ(0, source->Tell());