(tez) branch master updated: TEZ-4577: SortSpan could be created real small, resulting in eventual job failure (#367) (Chenyu Zheng reviewed by Laszlo Bodor)

[ab...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git


The following commit(s) were added to refs/heads/master by this push:
     new d84fdca2b TEZ-4577: SortSpan could be created real small, resulting in eventual job failure (#367) (Chenyu Zheng reviewed by Laszlo Bodor)
d84fdca2b is described below

commit d84fdca2b590793e08fc25b7e1ffc219293e3c28
Author: zhengchenyu <zh...@163.com>
AuthorDate: Thu Dec 12 19:28:12 2024 +0800

    TEZ-4577: SortSpan could be created real small, resulting in eventual job failure (#367) (Chenyu Zheng reviewed by Laszlo Bodor)
---
 .../apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index 1815006cd..d0eeadca0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -357,7 +357,7 @@ public class PipelinedSorter extends ExternalSorter {
       }
       Preconditions.checkArgument(buffers.get(bufferIndex) != null, "block should not be empty");
       //TODO: fix per item being passed.
-      span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), items,
+      span = new SortSpan((ByteBuffer)buffers.get(bufferIndex).clear(), (1024*1024),
           perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf));
     } else {
       // queue up the sort
@@ -940,7 +940,7 @@ public class PipelinedSorter extends ExternalSorter {
     public SortSpan(ByteBuffer source, int maxItems, int perItem, RawComparator comparator) {
       capacity = source.remaining();
       int metasize = METASIZE*maxItems;
-      int dataSize = maxItems * perItem;
+      long dataSize = (long) maxItems * (long) perItem;
       if(capacity < (metasize+dataSize)) {
         // try to allocate less meta space, because we have sample data
         metasize = METASIZE*(capacity/(perItem+METASIZE));