You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by zh...@apache.org on 2017/03/28 19:48:07 UTC
[26/50] [abbrv] tez git commit: TEZ-3647. Add a setting which lets
Tez determine Xmx. Contributed by Zhiyuan Yang.
TEZ-3647. Add a setting which lets Tez determine Xmx. Contributed by
Zhiyuan Yang.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1b1eb1d4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1b1eb1d4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1b1eb1d4
Branch: refs/heads/TEZ-1190
Commit: 1b1eb1d4b995c221d2eece2c12fca6ca52b94f27
Parents: 4ce6ea6
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Mar 3 13:45:05 2017 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Mar 3 13:45:05 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../main/java/org/apache/tez/client/TezClientUtils.java | 10 +++++++++-
.../java/org/apache/tez/dag/api/TezConfiguration.java | 4 ++++
.../main/java/org/apache/tez/dag/api/TezConstants.java | 6 ++++++
.../java/org/apache/tez/client/TestTezClientUtils.java | 10 ++++++++++
5 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 07841bf..763acd8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3647. Add a setting which lets Tez determine Xmx.
TEZ-3644. Cleanup container list stored in AMNode.
TEZ-3646. IFile.Writer has an extra output stream flush call
TEZ-3643. Long running AMs can go out of memory due to retained AMContainer instances.
http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index ab37b17..b16d1ca 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -39,6 +39,7 @@ import java.util.Map.Entry;
import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.math3.util.Precision;
import org.apache.tez.common.JavaOptsChecker;
import org.apache.tez.dag.api.records.DAGProtos.AMPluginDescriptorProto;
import org.apache.tez.serviceplugins.api.ServicePluginsDescriptor;
@@ -963,9 +964,16 @@ public class TezClientUtils {
|| (resource.getMemory() <= 0)) {
return javaOpts;
}
- if (maxHeapFactor <= 0 || maxHeapFactor >= 1) {
+
+ if ((maxHeapFactor <= 0 && !Precision.equals(maxHeapFactor, -1, 0.01)) || maxHeapFactor >= 1) {
return javaOpts;
}
+
+ if (Precision.equals(maxHeapFactor, -1, 0.01)) {
+ maxHeapFactor = resource.getMemory() < TezConstants.TEZ_CONTAINER_SMALL_SLAB_BOUND_MB
+ ? TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB
+ : TezConstants.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB;
+ }
int maxMemory = (int)(resource.getMemory() * maxHeapFactor);
maxMemory = maxMemory <= 0 ? 1 : maxMemory;
http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 94f40bb..5900f5e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -356,6 +356,10 @@ public class TezConfiguration extends Configuration {
* actual allocation of memory to tasks the cluster. The value if used as a
* fraction that is applied to the memory allocated Factor to size Xmx based
* on container memory size. Value should be greater than 0 and less than 1.
+ *
+ * Set this value to -1 to allow Tez to use different default max heap fraction
+ * for different container memory size. Current policy is to use 0.7 for container
+ * smaller than 4GB and use 0.8 for larger container.
*/
@ConfigurationScope(Scope.AM)
@ConfigurationProperty(type="float")
http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 06b9cb7..b75269a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -123,5 +123,11 @@ public class TezConstants {
*/
public static final long TEZ_AM_CLIENT_HEARTBEAT_POLL_INTERVAL_MILLIS_MINIMUM = 1000;
+ /**
+ * For container JVM Xmx auto determination
+ */
+ public static final int TEZ_CONTAINER_SMALL_SLAB_BOUND_MB = 4096;
+ public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_SMALL_SLAB = 0.7;
+ public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_LARGE_SLAB = 0.8;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/1b1eb1d4/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index a5e9d3c..7ff8125 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -629,6 +629,16 @@ public class TestTezClientUtils {
javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
Resource.newInstance(355, 1), 100);
Assert.assertEquals(origJavaOpts, javaOpts);
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(1000, 1), -1);
+ Assert.assertTrue(javaOpts.contains("-Xmx700m"));
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(5000, 1), -1);
+ Assert.assertTrue(javaOpts.contains("-Xmx4000m"));
}
@Test (timeout=5000)