You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "westonpace (via GitHub)" <gi...@apache.org> on 2023/02/16 22:14:42 UTC
[GitHub] [arrow] westonpace commented on a diff in pull request #14758: ARROW-18119: [C++] Utility method to ensure an array object meetings an alignment requirement
westonpace commented on code in PR #14758:
URL: https://github.com/apache/arrow/pull/14758#discussion_r1109067707
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
- std::vector<std::shared_ptr<Buffer>> buffers_ = object.data()->buffers;
+ auto buffer_address = object->address();
+ if ((buffer_address % alignment) != 0) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto new_buffer, AllocateBuffer(object->size(), alignment, memory_pool));
+ std::memcpy(new_buffer->mutable_data(), object->data(), object->size());
+ return new_buffer;
+ } else {
+ return object;
Review Comment:
```suggestion
return std::move(object);
```
Needed if you follow the previous advice and change to `std::shared_ptr<Buffer> object` instead of `const std::shared_ptr<Buffer>&`
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
Review Comment:
```suggestion
Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& buffer, int64_t alignment,
```
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
- std::vector<std::shared_ptr<Buffer>> buffers_ = object.data()->buffers;
+ auto buffer_address = object->address();
+ if ((buffer_address % alignment) != 0) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto new_buffer, AllocateBuffer(object->size(), alignment, memory_pool));
+ std::memcpy(new_buffer->mutable_data(), object->data(), object->size());
+ return new_buffer;
+ } else {
+ return object;
+ }
+}
+
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object, int64_t alignment,
+ MemoryPool* memory_pool) {
+ std::vector<std::shared_ptr<Buffer>> buffers_ = object->data()->buffers;
for (size_t i = 0; i < buffers_.size(); ++i) {
if (buffers_[i]) {
- auto buffer_address = buffers_[i]->address();
- if ((buffer_address % alignment) != 0) {
- ARROW_ASSIGN_OR_RAISE(
- auto new_buffer, AllocateBuffer(buffers_[i]->size(), alignment, memory_pool));
- std::memcpy(new_buffer->mutable_data(), buffers_[i]->data(), buffers_[i]->size());
- buffers_[i] = std::move(new_buffer);
+ ARROW_ASSIGN_OR_RAISE(buffers_[i], EnsureAlignment(buffers_[i], alignment, memory_pool));
}
- }
}
auto new_array_data =
- ArrayData::Make(object.data()->type, object.data()->length, std::move(buffers_),
- object.data()->GetNullCount(), object.data()->offset);
+ ArrayData::Make(object->data()->type, object->data()->length, std::move(buffers_),
+ object->data()->GetNullCount(), object->data()->offset);
return MakeArray(new_array_data);
}
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
+Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const std::shared_ptr<ChunkedArray>& object,
Review Comment:
```suggestion
Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const std::shared_ptr<ChunkedArray>& array,
```
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
Review Comment:
```suggestion
Result<std::shared_ptr<Buffer>> EnsureAlignment(std::shared_ptr<Buffer> object, int64_t alignment,
MemoryPool* memory_pool) {
```
Also, the method should use move appropriate so I can do...
```
std::shared_ptr<Buffer> foo = ...
foo = EnsureAlignment(std::move(foo), 64, default_memory_pool())
// Should only make a copy of `foo` or the `shared_ptr` if the buffer was not aligned.
```
##########
cpp/src/arrow/util/align_util.h:
##########
@@ -68,19 +68,25 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object,
Review Comment:
```suggestion
Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& array,
```
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
- std::vector<std::shared_ptr<Buffer>> buffers_ = object.data()->buffers;
+ auto buffer_address = object->address();
+ if ((buffer_address % alignment) != 0) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto new_buffer, AllocateBuffer(object->size(), alignment, memory_pool));
+ std::memcpy(new_buffer->mutable_data(), object->data(), object->size());
+ return new_buffer;
+ } else {
+ return object;
+ }
+}
+
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object, int64_t alignment,
+ MemoryPool* memory_pool) {
+ std::vector<std::shared_ptr<Buffer>> buffers_ = object->data()->buffers;
for (size_t i = 0; i < buffers_.size(); ++i) {
if (buffers_[i]) {
- auto buffer_address = buffers_[i]->address();
- if ((buffer_address % alignment) != 0) {
- ARROW_ASSIGN_OR_RAISE(
- auto new_buffer, AllocateBuffer(buffers_[i]->size(), alignment, memory_pool));
- std::memcpy(new_buffer->mutable_data(), buffers_[i]->data(), buffers_[i]->size());
- buffers_[i] = std::move(new_buffer);
+ ARROW_ASSIGN_OR_RAISE(buffers_[i], EnsureAlignment(buffers_[i], alignment, memory_pool));
}
- }
}
auto new_array_data =
- ArrayData::Make(object.data()->type, object.data()->length, std::move(buffers_),
- object.data()->GetNullCount(), object.data()->offset);
+ ArrayData::Make(object->data()->type, object->data()->length, std::move(buffers_),
Review Comment:
You still need to handle the dictionary and the children.
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
- std::vector<std::shared_ptr<Buffer>> buffers_ = object.data()->buffers;
+ auto buffer_address = object->address();
+ if ((buffer_address % alignment) != 0) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto new_buffer, AllocateBuffer(object->size(), alignment, memory_pool));
+ std::memcpy(new_buffer->mutable_data(), object->data(), object->size());
+ return new_buffer;
+ } else {
+ return object;
+ }
+}
+
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object, int64_t alignment,
+ MemoryPool* memory_pool) {
+ std::vector<std::shared_ptr<Buffer>> buffers_ = object->data()->buffers;
for (size_t i = 0; i < buffers_.size(); ++i) {
if (buffers_[i]) {
- auto buffer_address = buffers_[i]->address();
- if ((buffer_address % alignment) != 0) {
- ARROW_ASSIGN_OR_RAISE(
- auto new_buffer, AllocateBuffer(buffers_[i]->size(), alignment, memory_pool));
- std::memcpy(new_buffer->mutable_data(), buffers_[i]->data(), buffers_[i]->size());
- buffers_[i] = std::move(new_buffer);
+ ARROW_ASSIGN_OR_RAISE(buffers_[i], EnsureAlignment(buffers_[i], alignment, memory_pool));
}
- }
}
auto new_array_data =
- ArrayData::Make(object.data()->type, object.data()->length, std::move(buffers_),
- object.data()->GetNullCount(), object.data()->offset);
+ ArrayData::Make(object->data()->type, object->data()->length, std::move(buffers_),
+ object->data()->GetNullCount(), object->data()->offset);
return MakeArray(new_array_data);
}
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
+Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const std::shared_ptr<ChunkedArray>& object,
int64_t alignment,
MemoryPool* memory_pool) {
- ArrayVector chunks_ = object.chunks();
- for (int i = 0; i < object.num_chunks(); ++i) {
+ ArrayVector chunks_ = object->chunks();
+ for (int i = 0; i < object->num_chunks(); ++i) {
ARROW_ASSIGN_OR_RAISE(chunks_[i],
- EnsureAlignment(*object.chunk(i), alignment, memory_pool));
+ EnsureAlignment(object->chunk(i), alignment, memory_pool));
}
- return ChunkedArray::Make(std::move(chunks_), object.type());
+ return ChunkedArray::Make(std::move(chunks_), object->type());
}
-Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const RecordBatch& object,
+Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const std::shared_ptr<RecordBatch>& object,
Review Comment:
```suggestion
Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const std::shared_ptr<RecordBatch>& batch,
```
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
- std::vector<std::shared_ptr<Buffer>> buffers_ = object.data()->buffers;
+ auto buffer_address = object->address();
+ if ((buffer_address % alignment) != 0) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto new_buffer, AllocateBuffer(object->size(), alignment, memory_pool));
+ std::memcpy(new_buffer->mutable_data(), object->data(), object->size());
+ return new_buffer;
+ } else {
+ return object;
+ }
+}
+
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object, int64_t alignment,
+ MemoryPool* memory_pool) {
+ std::vector<std::shared_ptr<Buffer>> buffers_ = object->data()->buffers;
for (size_t i = 0; i < buffers_.size(); ++i) {
if (buffers_[i]) {
- auto buffer_address = buffers_[i]->address();
- if ((buffer_address % alignment) != 0) {
- ARROW_ASSIGN_OR_RAISE(
- auto new_buffer, AllocateBuffer(buffers_[i]->size(), alignment, memory_pool));
- std::memcpy(new_buffer->mutable_data(), buffers_[i]->data(), buffers_[i]->size());
- buffers_[i] = std::move(new_buffer);
+ ARROW_ASSIGN_OR_RAISE(buffers_[i], EnsureAlignment(buffers_[i], alignment, memory_pool));
}
- }
}
auto new_array_data =
- ArrayData::Make(object.data()->type, object.data()->length, std::move(buffers_),
- object.data()->GetNullCount(), object.data()->offset);
+ ArrayData::Make(object->data()->type, object->data()->length, std::move(buffers_),
+ object->data()->GetNullCount(), object->data()->offset);
return MakeArray(new_array_data);
}
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
+Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const std::shared_ptr<ChunkedArray>& object,
int64_t alignment,
MemoryPool* memory_pool) {
- ArrayVector chunks_ = object.chunks();
- for (int i = 0; i < object.num_chunks(); ++i) {
+ ArrayVector chunks_ = object->chunks();
+ for (int i = 0; i < object->num_chunks(); ++i) {
ARROW_ASSIGN_OR_RAISE(chunks_[i],
- EnsureAlignment(*object.chunk(i), alignment, memory_pool));
+ EnsureAlignment(object->chunk(i), alignment, memory_pool));
}
- return ChunkedArray::Make(std::move(chunks_), object.type());
+ return ChunkedArray::Make(std::move(chunks_), object->type());
}
-Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const RecordBatch& object,
+Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const std::shared_ptr<RecordBatch>& object,
int64_t alignment,
MemoryPool* memory_pool) {
- ArrayVector columns_ = object.columns();
- for (int i = 0; i < object.num_columns(); ++i) {
+ ArrayVector columns_ = object->columns();
+ for (int i = 0; i < object->num_columns(); ++i) {
ARROW_ASSIGN_OR_RAISE(columns_[i],
- EnsureAlignment(*object.column(i), alignment, memory_pool));
+ EnsureAlignment(object->column(i), alignment, memory_pool));
}
- return RecordBatch::Make(object.schema(), object.num_rows(), std::move(columns_));
+ return RecordBatch::Make(object->schema(), object->num_rows(), std::move(columns_));
}
-Result<std::shared_ptr<Table>> EnsureAlignment(const Table& object, int64_t alignment,
+Result<std::shared_ptr<Table>> EnsureAlignment(const std::shared_ptr<Table>& object, int64_t alignment,
Review Comment:
```suggestion
Result<std::shared_ptr<Table>> EnsureAlignment(const std::shared_ptr<Table>& table, int64_t alignment,
```
##########
cpp/src/arrow/util/align_util.h:
##########
@@ -68,19 +68,25 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const RecordBatch& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(
+ const std::shared_ptr<ChunkedArray>& object, int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<Table>> EnsureAlignment(const Table& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<RecordBatch>> EnsureAlignment(
+ const std::shared_ptr<RecordBatch>& object, int64_t alignment,
Review Comment:
```suggestion
const std::shared_ptr<RecordBatch>& batch, int64_t alignment,
```
##########
cpp/src/arrow/util/align_util.cc:
##########
@@ -26,56 +26,63 @@ namespace arrow {
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object, int64_t alignment,
MemoryPool* memory_pool) {
- std::vector<std::shared_ptr<Buffer>> buffers_ = object.data()->buffers;
+ auto buffer_address = object->address();
+ if ((buffer_address % alignment) != 0) {
+ ARROW_ASSIGN_OR_RAISE(
+ auto new_buffer, AllocateBuffer(object->size(), alignment, memory_pool));
+ std::memcpy(new_buffer->mutable_data(), object->data(), object->size());
+ return new_buffer;
+ } else {
+ return object;
+ }
+}
+
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object, int64_t alignment,
Review Comment:
```suggestion
Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& array, int64_t alignment,
```
##########
cpp/src/arrow/util/align_util.h:
##########
@@ -68,19 +68,25 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const RecordBatch& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(
+ const std::shared_ptr<ChunkedArray>& object, int64_t alignment,
Review Comment:
```suggestion
const std::shared_ptr<ChunkedArray>& array, int64_t alignment,
```
##########
cpp/src/arrow/util/align_util.h:
##########
@@ -68,19 +68,25 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object,
Review Comment:
```suggestion
Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& buffer,
```
##########
cpp/src/arrow/util/align_util.h:
##########
@@ -68,19 +68,25 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of
namespace util {
-Result<std::shared_ptr<Array>> EnsureAlignment(const Array& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Buffer>> EnsureAlignment(const std::shared_ptr<Buffer>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(const ChunkedArray& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<Array>> EnsureAlignment(const std::shared_ptr<Array>& object,
+ int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<RecordBatch>> EnsureAlignment(const RecordBatch& object,
- int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<ChunkedArray>> EnsureAlignment(
+ const std::shared_ptr<ChunkedArray>& object, int64_t alignment,
+ MemoryPool* memory_pool);
-Result<std::shared_ptr<Table>> EnsureAlignment(const Table& object, int64_t alignment,
- MemoryPool* memory_pool);
+Result<std::shared_ptr<RecordBatch>> EnsureAlignment(
+ const std::shared_ptr<RecordBatch>& object, int64_t alignment,
+ MemoryPool* memory_pool);
+
+Result<std::shared_ptr<Table>> EnsureAlignment(const std::shared_ptr<Table>& object,
Review Comment:
```suggestion
Result<std::shared_ptr<Table>> EnsureAlignment(const std::shared_ptr<Table>& table,
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org