You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2013/06/15 00:13:47 UTC
[1/2] git commit: TEZ-238. Add mapreduce-test-examples tests jar to
tez-dist tarball. (hitesh)
Updated Branches:
refs/heads/master b78dc0e0d -> e8c31ed67
TEZ-238. Add mapreduce-test-examples tests jar to tez-dist tarball. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/89e3c0b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/89e3c0b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/89e3c0b4
Branch: refs/heads/master
Commit: 89e3c0b474af227101db6772faadad72cb6d5ea6
Parents: b78dc0e
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Jun 14 14:15:40 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Jun 14 15:04:48 2013 -0700
----------------------------------------------------------------------
tez-dist/pom.xml | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/89e3c0b4/tez-dist/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml
index 0c3b9b6..4bda9a9 100644
--- a/tez-dist/pom.xml
+++ b/tez-dist/pom.xml
@@ -46,6 +46,12 @@
<artifactId>tez-mapreduce-examples</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-mapreduce-tests</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
<properties>
[2/2] git commit: TEZ-219. Create helper api to create environment
and java opts for Tez AM. (hitesh)
Posted by hi...@apache.org.
TEZ-219. Create helper api to create environment and java opts for Tez AM. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/e8c31ed6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/e8c31ed6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/e8c31ed6
Branch: refs/heads/master
Commit: e8c31ed679e2bfb8d7884e3e98125f3505f1f0af
Parents: 89e3c0b
Author: Hitesh Shah <hi...@apache.org>
Authored: Fri Jun 14 15:09:03 2013 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Fri Jun 14 15:09:03 2013 -0700
----------------------------------------------------------------------
.../apache/tez/dag/api/TezConfiguration.java | 44 +++++++--------
.../apache/tez/mapreduce/hadoop/MRHelpers.java | 57 ++++++++++++++++++++
.../tez/mapreduce/hadoop/TestMRHelpers.java | 29 ++++++++++
3 files changed, 108 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e8c31ed6/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 498e3fb..6f915be 100644
--- a/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-dag-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -38,79 +38,79 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_PREFIX = "tez.";
public static final String TEZ_AM_PREFIX = TEZ_PREFIX + "dag.am.";
-
+
public static final String DAG_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
public static final String DAG_AM_STAGING_DIR_DEFAULT = "/tmp/hadoop-yarn/staging";
-
+
// TODO Should not be required once all tokens are handled via AppSubmissionContext
public static final String JOB_SUBMIT_DIR = TEZ_PREFIX + "jobSubmitDir";
public static final String APPLICATION_TOKENS_FILE = "appTokens";
- public static final String DAG_APPLICATION_MASTER_CLASS =
+ public static final String DAG_APPLICATION_MASTER_CLASS =
"org.apache.tez.dag.app.DAGAppMaster";
/** Root Logging level passed to the Tez app master.*/
public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX+"log.level";
public static final String DEFAULT_TEZ_AM_LOG_LEVEL = "INFO";
-
- public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX +
+
+ public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX +
"am.complete.cancel.delegation.tokens";
public static final boolean DEFAULT_TEZ_AM_CANCEL_DELEGATION_TOKEN = true;
-
- public static final String DAG_AM_TASK_LISTENER_THREAD_COUNT =
+
+ public static final String DAG_AM_TASK_LISTENER_THREAD_COUNT =
TEZ_PREFIX + "task.listener.thread-count";
public static final int DAG_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
- public static final String DAG_AM_CONTAINER_LISTENER_THREAD_COUNT =
+ public static final String DAG_AM_CONTAINER_LISTENER_THREAD_COUNT =
TEZ_PREFIX + "container.listener.thread-count";
public static final int DAG_AM_CONTAINER_LISTENER_THREAD_COUNT_DEFAULT = 30;
-
+
// TODO Some of the DAG properties are job specific and not AM specific. Rename accordingly.
// TODO Are any of these node blacklisting properties required. (other than for MR compat)
public static final String DAG_MAX_TASK_FAILURES_PER_NODE = TEZ_PREFIX
+ "maxtaskfailures.per.node";
public static final int DAG_MAX_TASK_FAILURES_PER_NODE_DEFAULT = 3;
-
- public static final String DAG_MAX_TASK_ATTEMPTS =
+
+ public static final String DAG_MAX_TASK_ATTEMPTS =
TEZ_AM_PREFIX + "max.task.attempts";
public static final int DAG_MAX_TASK_ATTEMPTS_DEFAULT = 4;
-
+
public static final String DAG_NODE_BLACKLISTING_ENABLED = TEZ_PREFIX
+ "node-blacklisting.enabled";
public static final boolean DAG_NODE_BLACKLISTING_ENABLED_DEFAULT = true;
public static final String DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD = TEZ_PREFIX
+ "node-blacklisting.ignore-threshold-node-percent";
public static final int DAG_NODE_BLACKLISTING_IGNORE_THRESHOLD_DEFAULT = 33;
-
+
/** Number of threads to handle job client RPC requests.*/
public static final String DAG_CLIENT_AM_THREAD_COUNT =
TEZ_PREFIX + "client.am.thread-count";
public static final int DAG_CLIENT_AM__THREAD_COUNT_DEFAULT = 1;
- /**
+ /**
* Range of ports that the AM can use when binding. Leave blank
* if you want all possible ports.
*/
- public static final String DAG_CLIENT_AM_PORT_RANGE =
+ public static final String DAG_CLIENT_AM_PORT_RANGE =
TEZ_PREFIX + "client.am.port-range";
public static final String DAG_AM_RESOURCE_MEMORY_MB = TEZ_AM_PREFIX
+ "resource.memory.mb";
- public static final int DEFAULT_DAG_AM_RESOURCE_MEMORY_MB = 1024;
+ public static final int DEFAULT_DAG_AM_RESOURCE_MEMORY_MB = 1536;
public static final String DAG_AM_RESOURCE_CPU_VCORES = TEZ_AM_PREFIX
+ "resource.cpu.vcores";
public static final int DEFAULT_DAG_AM_RESOURCE_CPU_VCORES = 1;
-
- public static final String
+
+ public static final String
SLOWSTART_VERTEX_SCHEDULER_MIN_SRC_FRACTION = TEZ_PREFIX
+ "slowstart-vertex-scheduler.min-src-fraction";
- public static final float
+ public static final float
SLOWSTART_VERTEX_SCHEDULER_MIN_SRC_FRACTION_DEFAULT = 1f;
- public static final String
+ public static final String
SLOWSTART_VERTEX_SCHEDULER_MAX_SRC_FRACTION = TEZ_PREFIX
+ "slowstart-vertex-scheduler.max-src-fraction";
- public static final float
+ public static final float
SLOWSTART_VERTEX_SCHEDULER_MAX_SRC_FRACTION_DEFAULT = 1f;
/**
@@ -127,6 +127,6 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_LIB_URIS =
TEZ_PREFIX + "lib.uris";
-
+
public static final String TEZ_APPLICATION_TYPE = "TEZ-MR*";
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e8c31ed6/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
index 475b3a0..b84a2e3 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRHelpers.java
@@ -24,6 +24,7 @@ import java.net.InetAddress;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Vector;
@@ -44,6 +45,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobSubmissionFiles;
import org.apache.hadoop.mapreduce.split.JobSplitWriter;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -595,4 +597,59 @@ public class MRHelpers {
getChildLogLevel(conf, isMap) + ",CLA");
}
+ private static Configuration getBaseJobConf(Configuration conf) {
+ if (conf != null) {
+ return new JobConf(conf);
+ } else {
+ return new JobConf();
+ }
+ }
+
+ /**
+ * Get default initialize JobConf-based configuration
+ * @param conf Conf to initialize JobConf with.
+ * @return Base configuration for MR-based jobs
+ */
+ public static Configuration getBaseMRConfiguration(Configuration conf) {
+ return getBaseJobConf(conf);
+ }
+
+ /**
+ * Get default initialize JobConf-based configuration
+ * @return Base configuration for MR-based jobs
+ */
+ public static Configuration getBaseMRConfiguration() {
+ return getBaseJobConf(null);
+ }
+
+ /**
+ * Setup environment for the AM based on MR-based configuration
+ * @param conf Configuration from which to extract information
+ * @param environment Environment map to update
+ */
+ public static void updateEnvironmentForMRAM(Configuration conf,
+ Map<String, String> environment) {
+ MRApps.setEnvFromInputString(environment,
+ conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV));
+ MRApps.setEnvFromInputString(environment,
+ conf.get(MRJobConfig.MR_AM_ENV));
+ }
+
+ /**
+ * Extract Java Opts for the AM based on MR-based configuration
+ * @param conf Configuration from which to extract information
+ * @return Java opts for the AM
+ */
+ public static String getMRAMJavaOpts(Configuration conf) {
+ // Admin opts
+ String mrAppMasterAdminOptions = conf.get(
+ MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS);
+ // Add AM user command opts
+ String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS,
+ MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS);
+
+ return mrAppMasterAdminOptions.trim()
+ + " " + mrAppMasterUserOptions.trim();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e8c31ed6/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
index 9158280..ac0ed45 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestMRHelpers.java
@@ -357,4 +357,33 @@ public class TestMRHelpers {
Assert.assertEquals("red1", env.get("foo"));
Assert.assertEquals("red2", env.get("bar"));
}
+
+ @Test
+ public void testGetBaseMRConf() {
+ Configuration conf = MRHelpers.getBaseMRConfiguration();
+ Assert.assertNotNull(conf);
+ conf = MRHelpers.getBaseMRConfiguration(new YarnConfiguration());
+ Assert.assertNotNull(conf);
+ }
+
+ @Test
+ public void testMRAMJavaOpts() {
+ Configuration conf = new Configuration();
+ conf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, " -Dadminfoobar ");
+ conf.set(MRJobConfig.MR_AM_COMMAND_OPTS, " -Duserfoo ");
+ String opts = MRHelpers.getMRAMJavaOpts(conf);
+ Assert.assertEquals("-Dadminfoobar -Duserfoo", opts);
+ }
+
+ public void testMRAMEnvironmentSetup() {
+ Configuration conf = new Configuration();
+ conf.set(MRJobConfig.MR_AM_ADMIN_USER_ENV, "foo=bar,admin1=foo1");
+ conf.set(MRJobConfig.MR_AM_ENV, "foo=bar2,user=foo2");
+ Map<String, String> env =
+ new HashMap<String, String>();
+ MRHelpers.updateEnvironmentForMRAM(conf, env);
+ Assert.assertEquals("foo1", env.get("admin1"));
+ Assert.assertEquals("foo2", env.get("user"));
+ Assert.assertEquals("bar2", env.get("foo"));
+ }
}