You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2014/06/28 01:18:24 UTC
git commit: TEZ-699. Have sensible defaults for java opts. (hitesh)
Repository: incubator-tez
Updated Branches:
refs/heads/master df7ff3cb4 -> 7aa927a72
TEZ-699. Have sensible defaults for java opts. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/7aa927a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/7aa927a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/7aa927a7
Branch: refs/heads/master
Commit: 7aa927a72f8142a5d6e05531fd948dbff85493a9
Parents: df7ff3c
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Jun 27 16:17:55 2014 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Jun 27 16:17:55 2014 -0700
----------------------------------------------------------------------
.../org/apache/tez/client/TezClientUtils.java | 34 +++++++++--
.../main/java/org/apache/tez/dag/api/DAG.java | 6 +-
.../apache/tez/dag/api/TezConfiguration.java | 15 +++--
.../apache/tez/client/TestTezClientUtils.java | 59 ++++++++++++++++++++
.../app/rm/container/AMContainerHelpers.java | 22 ++++++--
.../dag/app/rm/container/AMContainerImpl.java | 3 +-
.../dag/app/rm/container/TestAMContainer.java | 3 +
7 files changed, 125 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index bae0328..dd1b997 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -314,7 +314,7 @@ public class TezClientUtils {
* @param amName Name for the application
* @param amConfig AM Configuration
* @param tezJarResources Resources to be used by the AM
- * @param sessionCredentials the credential object which will be populated with session specific
+ * @param sessionCreds the credential object which will be populated with session specific
* @return an ApplicationSubmissionContext to launch a Tez AM
* @throws IOException
* @throws YarnException
@@ -377,14 +377,14 @@ public class TezClientUtils {
String amOpts = amConfig.getAMConf().get(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS,
TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT);
- if (amOpts != null && !amOpts.isEmpty()) {
- vargs.add(amOpts);
- }
+ amOpts = maybeAddDefaultMemoryJavaOpts(amOpts, capability,
+ amConfig.getAMConf().getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
+ TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+ vargs.add(amOpts);
String amLogLevel = amConfig.getAMConf().get(
TezConfiguration.TEZ_AM_LOG_LEVEL,
TezConfiguration.TEZ_AM_LOG_LEVEL_DEFAULT);
-
maybeAddDefaultLoggingJavaOpts(amLogLevel, vargs);
// FIX sun bug mentioned in TEZ-327
@@ -805,4 +805,28 @@ public class TezClientUtils {
TokenCache.setSessionToken(sessionToken, credentials);
}
+ /**
+ * Add computed Xmx value to java opts if both -Xms and -Xmx are not specified
+ * @param javaOpts Current java opts
+ * @param resource Resource capability based on which java opts will be computed
+ * @param maxHeapFactor Factor to size Xmx ( valid range is 0.0 < x < 1.0)
+ * @return Modified java opts with computed Xmx value
+ */
+ public static String maybeAddDefaultMemoryJavaOpts(String javaOpts, Resource resource,
+ double maxHeapFactor) {
+ if ((javaOpts != null && !javaOpts.isEmpty()
+ && (javaOpts.contains("-Xmx") || javaOpts.contains("-Xms")))
+ || (resource.getMemory() <= 0)) {
+ return javaOpts;
+ }
+ if (maxHeapFactor <= 0 || maxHeapFactor >= 1) {
+ return javaOpts;
+ }
+ int maxMemory = (int)(resource.getMemory() * maxHeapFactor);
+ maxMemory = maxMemory <= 0 ? 1 : maxMemory;
+
+ return " -Xmx" + maxMemory + "m "
+ + ( javaOpts != null ? javaOpts : "");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 9df37b3..4772fae 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -62,7 +62,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-public class DAG { // FIXME rename to Topology
+public class DAG {
private static final Log LOG = LogFactory.getLog(DAG.class);
@@ -498,9 +498,9 @@ public class DAG { // FIXME rename to Topology
// there was something on the stack other than this "av".
// this indicates there is a scc/cycle. It comprises all nodes from top of stack to "av"
StringBuilder message = new StringBuilder();
- message.append(av.v.getVertexName() + " <- ");
+ message.append(av.v.getVertexName()).append(" <- ");
for (; pop != av; pop = stack.pop()) {
- message.append(pop.v.getVertexName() + " <- ");
+ message.append(pop.v.getVertexName()).append(" <- ");
pop.onstack = false;
}
message.append(av.v.getVertexName());
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 8c1bf8a..610f919 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -66,16 +66,22 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_LAUNCH_CMD_OPTS = TEZ_AM_PREFIX + "java.opts";
public static final String TEZ_AM_LAUNCH_CMD_OPTS_DEFAULT =
"-Djava.net.preferIPv4Stack=true " +
- "-Dhadoop.metrics.log.level=WARN " +
- "-Xmx1024m"; // Remove after TEZ-699
+ "-Dhadoop.metrics.log.level=WARN ";
/** Command line options for the Tez Task processes. */
public static final String TEZ_TASK_LAUNCH_CMD_OPTS = TEZ_TASK_PREFIX
+ "launch.cmd-opts";
public static final String TEZ_TASK_LAUNCH_CMD_OPTS_DEFAULT =
"-Djava.net.preferIPv4Stack=true " +
- "-Dhadoop.metrics.log.level=WARN " +
- "-Xmx200m"; // Remove after TEZ-699
+ "-Dhadoop.metrics.log.level=WARN ";
+
+ /**
+ * Factor to size Xmx based on container memory size. Value should be greater than 0 and
+ * less than 1.
+ */
+ public static final String TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION =
+ TEZ_PREFIX + "container.max.java.heap.fraction";
+ public static final double TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT = 0.8;
/** Env settings for the Tez AppMaster process.
* Should be specified as a comma-separated of key-value pairs where each pair
@@ -478,4 +484,5 @@ public class TezConfiguration extends Configuration {
* The maximium number of tasks running in parallel in inline mode. Not valid till Tez-684 get checked-in
*/
public static final int TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT = 1;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 1762a8a..e519187 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -19,6 +19,7 @@ package org.apache.tez.client;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileNotFoundException;
@@ -31,6 +32,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.junit.Assert;
@@ -61,6 +63,7 @@ public class TestTezClientUtils {
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesDefinedButEmpty() throws Exception {
File emptyDir = new File(TEST_ROOT_DIR, "emptyDir");
+ emptyDir.deleteOnExit();
Assert.assertTrue(emptyDir.mkdirs());
Path emptyDirPath = new Path(emptyDir.getAbsolutePath());
TezConfiguration conf = new TezConfiguration();
@@ -159,4 +162,60 @@ public class TestTezClientUtils {
Map<String, LocalResource> localizedMap = TezClientUtils.setupTezJarsLocalResources(conf, credentials);
assertFalse(localizedMap.isEmpty());
}
+
+
+ @Test (timeout=5000)
+ public void testDefaultMemoryJavaOpts() {
+ final double factor = 0.8;
+ String origJavaOpts = "-Xmx";
+ String javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(1000, 1), factor);
+ Assert.assertEquals(origJavaOpts, javaOpts);
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(1000, 1), factor);
+ Assert.assertTrue(javaOpts.contains("-Xmx800m"));
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(1, 1), factor);
+ Assert.assertTrue(javaOpts.contains("-Xmx1m"));
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(-1, 1), factor);
+ Assert.assertEquals(origJavaOpts, javaOpts);
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(355, 1), factor);
+ Assert.assertTrue(javaOpts.contains("-Xmx284m"));
+
+ origJavaOpts = " -Xms100m ";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(355, 1), factor);
+ Assert.assertFalse(javaOpts.contains("-Xmx284m"));
+ Assert.assertTrue(javaOpts.contains("-Xms100m"));
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(355, 1), 0);
+ Assert.assertEquals(origJavaOpts, javaOpts);
+
+ origJavaOpts = "";
+ javaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(origJavaOpts,
+ Resource.newInstance(355, 1), 100);
+ Assert.assertEquals(origJavaOpts, javaOpts);
+ }
+
+ @Test (timeout=5000)
+ public void testDefaultLoggingJavaOpts() {
+ String origJavaOpts = null;
+ String javaOpts = TezClientUtils.maybeAddDefaultLoggingJavaOpts("FOOBAR", origJavaOpts);
+ Assert.assertNotNull(javaOpts);
+ Assert.assertTrue(javaOpts.contains("-D" + TezConfiguration.TEZ_ROOT_LOGGER_NAME + "=FOOBAR")
+ && javaOpts.contains(TezConfiguration.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
index b569fbd..db776a4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerHelpers.java
@@ -29,6 +29,7 @@ import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -41,8 +42,10 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.TezClientUtils;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.TokenCache;
import org.apache.tez.dag.api.TezConfiguration;
@@ -98,8 +101,6 @@ public class AMContainerHelpers {
// Service data
Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
-
-
// Tokens
// Setup up task credentials buffer
@@ -145,7 +146,8 @@ public class AMContainerHelpers {
Map<String, String> vertexEnv,
String javaOpts,
InetSocketAddress taskAttemptListenerAddress, Credentials credentials,
- AppContext appContext) {
+ AppContext appContext, Resource containerResource,
+ Configuration conf) {
ContainerLaunchContext commonContainerSpec = null;
synchronized (commonContainerSpecLock) {
@@ -180,10 +182,22 @@ public class AMContainerHelpers {
myEnv.putAll(env);
myEnv.putAll(vertexEnv);
+ String modifiedJavaOpts = TezClientUtils.maybeAddDefaultMemoryJavaOpts(javaOpts,
+ containerResource, conf.getDouble(TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION,
+ TezConfiguration.TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION_DEFAULT));
+ if (LOG.isDebugEnabled()) {
+ if (!modifiedJavaOpts.equals(javaOpts)) {
+ LOG.debug("Modified java opts for container"
+ + ", containerId=" + containerId
+ + ", originalJavaOpts=" + javaOpts
+ + ", modifiedJavaOpts=" + modifiedJavaOpts);
+ }
+ }
+
List<String> commands = TezRuntimeChildJVM.getVMCommand(
taskAttemptListenerAddress, containerId.toString(),
appContext.getApplicationID().toString(),
- appContext.getApplicationAttemptId().getAttemptId(), javaOpts);
+ appContext.getApplicationAttemptId().getAttemptId(), modifiedJavaOpts);
// Duplicate the ByteBuffers for access by multiple containers.
Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
index ae57658..9d36c58 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/container/AMContainerImpl.java
@@ -504,7 +504,8 @@ public class AMContainerImpl implements AMContainer {
containerContext.getEnvironment(),
containerContext.getJavaOpts(),
container.taskAttemptListener.getAddress(), containerContext.getCredentials(),
- container.appContext);
+ container.appContext, container.container.getResource(),
+ container.appContext.getAMConf());
// Registering now, so that in case of delayed NM response, the child
// task is not told to die since the TAL does not know about the container.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/7aa927a7/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
index 0862116..3b3fd37 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/container/TestAMContainer.java
@@ -39,6 +39,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent;
import org.apache.hadoop.security.Credentials;
@@ -1039,6 +1040,7 @@ public class TestAMContainer {
eventHandler = mock(EventHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
+ Configuration conf = new Configuration(false);
appContext = mock(AppContext.class);
doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext)
.getApplicationACLs();
@@ -1047,6 +1049,7 @@ public class TestAMContainer {
doReturn(applicationID).when(appContext).getApplicationID();
doReturn(new SystemClock()).when(appContext).getClock();
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
+ doReturn(conf).when(appContext).getAMConf();
mockDAGID();
taskSpec = mock(TaskSpec.class);