You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by wa...@apache.org on 2025/10/23 02:30:03 UTC
(incubator-gluten) branch main updated: [GLUTEN-10920][VL] Allow disabling hash/sort shuffle reader buffer (#10922)
This is an automated email from the ASF dual-hosted git repository.
wangzhen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 7b7ef95c65 [GLUTEN-10920][VL] Allow disabling hash/sort shuffle reader buffer (#10922)
7b7ef95c65 is described below
commit 7b7ef95c65cf911f03dc9bc7ed55c07fc408a03b
Author: Zhen Wang <64...@qq.com>
AuthorDate: Thu Oct 23 10:29:58 2025 +0800
[GLUTEN-10920][VL] Allow disabling hash/sort shuffle reader buffer (#10922)
---
cpp/velox/shuffle/VeloxShuffleReader.cc | 24 ++++++++++++++++--------
1 file changed, 16 insertions(+), 8 deletions(-)
diff --git a/cpp/velox/shuffle/VeloxShuffleReader.cc b/cpp/velox/shuffle/VeloxShuffleReader.cc
index 00e2225744..5e7b78f00d 100644
--- a/cpp/velox/shuffle/VeloxShuffleReader.cc
+++ b/cpp/velox/shuffle/VeloxShuffleReader.cc
@@ -501,10 +501,14 @@ void VeloxHashShuffleReaderDeserializer::loadNextStream() {
return;
}
- GLUTEN_ASSIGN_OR_THROW(
- in_,
- arrow::io::BufferedInputStream::Create(
- readerBufferSize_, memoryManager_->defaultArrowMemoryPool(), std::move(in)));
+ if (readerBufferSize_ > 0) {
+ GLUTEN_ASSIGN_OR_THROW(
+ in_,
+ arrow::io::BufferedInputStream::Create(
+ readerBufferSize_, memoryManager_->defaultArrowMemoryPool(), std::move(in)));
+ } else {
+ in_ = std::move(in);
+ }
}
std::shared_ptr<ColumnarBatch> VeloxHashShuffleReaderDeserializer::next() {
@@ -656,10 +660,14 @@ void VeloxSortShuffleReaderDeserializer::loadNextStream() {
GLUTEN_ASSIGN_OR_THROW(
in_, CompressedInputStream::Make(codec_.get(), std::move(in), memoryManager_->defaultArrowMemoryPool()));
} else {
- GLUTEN_ASSIGN_OR_THROW(
- in_,
- arrow::io::BufferedInputStream::Create(
- readerBufferSize_, memoryManager_->defaultArrowMemoryPool(), std::move(in)));
+ if (readerBufferSize_ > 0) {
+ GLUTEN_ASSIGN_OR_THROW(
+ in_,
+ arrow::io::BufferedInputStream::Create(
+ readerBufferSize_, memoryManager_->defaultArrowMemoryPool(), std::move(in)));
+ } else {
+ in_ = std::move(in);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org