You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/02/21 02:17:55 UTC
git commit: TEZ-835. Fix a bug which could cause a divide by zero if
a very small sort buffer is configured. (sseth)
Repository: incubator-tez
Updated Branches:
refs/heads/master 4f98537d6 -> 8884ee446
TEZ-835. Fix a bug which could cause a divide by zero if a very small
sort buffer is configured. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/8884ee44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/8884ee44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/8884ee44
Branch: refs/heads/master
Commit: 8884ee44639db284b783a255fb85eb3d02003544
Parents: 4f98537
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Feb 20 17:17:01 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Feb 20 17:17:01 2014 -0800
----------------------------------------------------------------------
.../library/common/sort/impl/ExternalSorter.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/8884ee44/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
index 78e55c1..3e3b25f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
@@ -53,6 +53,8 @@ import org.apache.tez.runtime.library.common.sort.impl.IFile.Writer;
import org.apache.tez.runtime.library.common.task.local.output.TezTaskOutput;
import org.apache.tez.runtime.library.hadoop.compat.NullProgressable;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings({"unchecked", "rawtypes"})
public abstract class ExternalSorter implements MemoryUpdateCallback {
@@ -64,6 +66,7 @@ public abstract class ExternalSorter implements MemoryUpdateCallback {
public abstract void write(Object key, Object value) throws IOException;
+ private int initialMemRequestMb;
protected Progressable nullProgressable = new NullProgressable();
protected TezOutputContext outputContext;
protected Combiner combiner;
@@ -105,13 +108,14 @@ public abstract class ExternalSorter implements MemoryUpdateCallback {
rfs = ((LocalFileSystem)FileSystem.getLocal(this.conf)).getRaw();
- int reqMemory =
+ initialMemRequestMb =
this.conf.getInt(
TezJobConfig.TEZ_RUNTIME_IO_SORT_MB,
TezJobConfig.DEFAULT_TEZ_RUNTIME_IO_SORT_MB);
- long reqBytes = reqMemory << 20;
+ Preconditions.checkArgument(initialMemRequestMb != 0, "io.sort.mb should be larger than 0");
+ long reqBytes = initialMemRequestMb << 20;
outputContext.requestInitialMemory(reqBytes, this);
- LOG.info("Requested SortBufferSize (io.sort.mb): " + reqMemory);
+ LOG.info("Requested SortBufferSize (io.sort.mb): " + initialMemRequestMb);
// sorter
sorter = ReflectionUtils.newInstance(this.conf.getClass(
@@ -232,5 +236,10 @@ public abstract class ExternalSorter implements MemoryUpdateCallback {
@Override
public void memoryAssigned(long assignedSize) {
this.availableMemoryMb = (int) (assignedSize >> 20);
+ if (this.availableMemoryMb == 0) {
+ LOG.warn("AssignedMemoryMB: " + this.availableMemoryMb
+ + " is too low. Falling back to initial ask: " + initialMemRequestMb);
+ this.availableMemoryMb = initialMemRequestMb;
+ }
}
}