You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ma...@apache.org on 2025/03/28 11:55:08 UTC

(incubator-gluten) branch main updated: [VL] LocalPartitionWriter should discard the evict since it use hash/sort evict (#9167)

This is an automated email from the ASF dual-hosted git repository.

marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 38e6543109 [VL] LocalPartitionWriter should discard the evict since it use hash/sort evict (#9167)
38e6543109 is described below

commit 38e6543109ad4ae04327a3c51e5c52dbcfd5c059
Author: Kaifei Yi <yi...@apache.org>
AuthorDate: Fri Mar 28 19:55:05 2025 +0800

    [VL] LocalPartitionWriter should discard the evict since it use hash/sort evict (#9167)
---
 cpp/core/shuffle/LocalPartitionWriter.cc | 26 --------------------------
 cpp/core/shuffle/LocalPartitionWriter.h  |  5 ++++-
 2 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/cpp/core/shuffle/LocalPartitionWriter.cc b/cpp/core/shuffle/LocalPartitionWriter.cc
index 183b688f7c..b0d1b49aeb 100644
--- a/cpp/core/shuffle/LocalPartitionWriter.cc
+++ b/cpp/core/shuffle/LocalPartitionWriter.cc
@@ -618,32 +618,6 @@ arrow::Status LocalPartitionWriter::sortEvict(
   return arrow::Status::OK();
 }
 
-// FIXME: Remove this code path for local partition writer.
-arrow::Status LocalPartitionWriter::evict(uint32_t partitionId, std::unique_ptr<BlockPayload> blockPayload, bool stop) {
-  rawPartitionLengths_[partitionId] += blockPayload->rawSize();
-
-  if (lastEvictPid_ != -1 && partitionId < lastEvictPid_) {
-    RETURN_NOT_OK(finishSpill(true));
-    lastEvictPid_ = -1;
-  }
-  RETURN_NOT_OK(requestSpill(stop));
-
-  if (!stop) {
-    RETURN_NOT_OK(spiller_->spill(partitionId, std::move(blockPayload)));
-  } else {
-    if (spills_.size() > 0) {
-      for (auto pid = lastEvictPid_ + 1; pid <= partitionId; ++pid) {
-        auto bytesEvicted = totalBytesEvicted_;
-        RETURN_NOT_OK(mergeSpills(pid));
-        partitionLengths_[pid] = totalBytesEvicted_ - bytesEvicted;
-      }
-    }
-    RETURN_NOT_OK(spiller_->spill(partitionId, std::move(blockPayload)));
-  }
-  lastEvictPid_ = partitionId;
-  return arrow::Status::OK();
-}
-
 arrow::Status LocalPartitionWriter::reclaimFixedSize(int64_t size, int64_t* actual) {
   // Finish last spiller.
   RETURN_NOT_OK(finishSpill(true));
diff --git a/cpp/core/shuffle/LocalPartitionWriter.h b/cpp/core/shuffle/LocalPartitionWriter.h
index 3ed0a13060..de1bce8e74 100644
--- a/cpp/core/shuffle/LocalPartitionWriter.h
+++ b/cpp/core/shuffle/LocalPartitionWriter.h
@@ -47,7 +47,10 @@ class LocalPartitionWriter : public PartitionWriter {
       std::shared_ptr<arrow::Buffer> compressed,
       bool isFinal) override;
 
-  arrow::Status evict(uint32_t partitionId, std::unique_ptr<BlockPayload> blockPayload, bool stop) override;
+  // This code path is not used by LocalPartitionWriter, Not implement it by default.
+  arrow::Status evict(uint32_t partitionId, std::unique_ptr<BlockPayload> blockPayload, bool stop) override {
+    return arrow::Status::NotImplemented("Invalid code path for local shuffle writer.");
+  }
 
   /// The stop function performs several tasks:
   /// 1. Opens the final data file.


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org