You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ma...@apache.org on 2025/03/28 11:55:08 UTC
(incubator-gluten) branch main updated: [VL] LocalPartitionWriter should discard the evict since it use hash/sort evict (#9167)
This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 38e6543109 [VL] LocalPartitionWriter should discard the evict since it use hash/sort evict (#9167)
38e6543109 is described below
commit 38e6543109ad4ae04327a3c51e5c52dbcfd5c059
Author: Kaifei Yi <yi...@apache.org>
AuthorDate: Fri Mar 28 19:55:05 2025 +0800
[VL] LocalPartitionWriter should discard the evict since it use hash/sort evict (#9167)
---
cpp/core/shuffle/LocalPartitionWriter.cc | 26 --------------------------
cpp/core/shuffle/LocalPartitionWriter.h | 5 ++++-
2 files changed, 4 insertions(+), 27 deletions(-)
diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc b/cpp/core/shuffle/LocalPartitionWriter.cc
index 183b688f7c..b0d1b49aeb 100644
--- a/cpp/core/shuffle/LocalPartitionWriter.cc
+++ b/cpp/core/shuffle/LocalPartitionWriter.cc
@@ -618,32 +618,6 @@ arrow::Status LocalPartitionWriter::sortEvict(
return arrow::Status::OK();
}
-// FIXME: Remove this code path for local partition writer.
-arrow::Status LocalPartitionWriter::evict(uint32_t partitionId, std::unique_ptr<BlockPayload> blockPayload, bool stop) {
- rawPartitionLengths_[partitionId] += blockPayload->rawSize();
-
- if (lastEvictPid_ != -1 && partitionId < lastEvictPid_) {
- RETURN_NOT_OK(finishSpill(true));
- lastEvictPid_ = -1;
- }
- RETURN_NOT_OK(requestSpill(stop));
-
- if (!stop) {
- RETURN_NOT_OK(spiller_->spill(partitionId, std::move(blockPayload)));
- } else {
- if (spills_.size() > 0) {
- for (auto pid = lastEvictPid_ + 1; pid <= partitionId; ++pid) {
- auto bytesEvicted = totalBytesEvicted_;
- RETURN_NOT_OK(mergeSpills(pid));
- partitionLengths_[pid] = totalBytesEvicted_ - bytesEvicted;
- }
- }
- RETURN_NOT_OK(spiller_->spill(partitionId, std::move(blockPayload)));
- }
- lastEvictPid_ = partitionId;
- return arrow::Status::OK();
-}
-
arrow::Status LocalPartitionWriter::reclaimFixedSize(int64_t size, int64_t* actual) {
// Finish last spiller.
RETURN_NOT_OK(finishSpill(true));
diff --git a/cpp/core/shuffle/LocalPartitionWriter.h b/cpp/core/shuffle/LocalPartitionWriter.h
index 3ed0a13060..de1bce8e74 100644
--- a/cpp/core/shuffle/LocalPartitionWriter.h
+++ b/cpp/core/shuffle/LocalPartitionWriter.h
@@ -47,7 +47,10 @@ class LocalPartitionWriter : public PartitionWriter {
std::shared_ptr<arrow::Buffer> compressed,
bool isFinal) override;
- arrow::Status evict(uint32_t partitionId, std::unique_ptr<BlockPayload> blockPayload, bool stop) override;
+ // This code path is not used by LocalPartitionWriter, Not implement it by default.
+ arrow::Status evict(uint32_t partitionId, std::unique_ptr<BlockPayload> blockPayload, bool stop) override {
+ return arrow::Status::NotImplemented("Invalid code path for local shuffle writer.");
+ }
/// The stop function performs several tasks:
/// 1. Opens the final data file.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org