You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/11/05 22:52:48 UTC
git commit: TEZ-593. Add a config parameter to specify the YARN queue
name for a DAG. (sseth)
Updated Branches:
refs/heads/master 4eabbf714 -> 83a657bb5
TEZ-593. Add a config parameter to specify the YARN queue name for a
DAG. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/83a657bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/83a657bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/83a657bb
Branch: refs/heads/master
Commit: 83a657bb5d88c1d080d45b19d5243990cfb67efa
Parents: 4eabbf7
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Nov 5 13:52:29 2013 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Nov 5 13:52:29 2013 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/tez/client/AMConfiguration.java | 4 ++--
.../src/main/java/org/apache/tez/client/TezClientUtils.java | 4 +++-
.../src/main/java/org/apache/tez/dag/api/TezConfiguration.java | 6 ++++++
.../org/apache/tez/mapreduce/examples/FilterLinesByWord.java | 2 +-
.../java/org/apache/tez/mapreduce/examples/MRRSleepJob.java | 2 +-
.../org/apache/tez/mapreduce/examples/OrderedWordCount.java | 2 +-
.../main/java/org/apache/tez/mapreduce/client/YARNRunner.java | 5 +++--
.../java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 2 ++
.../test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java | 4 ++--
9 files changed, 21 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
index f452c74..3dd6424 100644
--- a/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/client/AMConfiguration.java
@@ -38,15 +38,15 @@ public class AMConfiguration {
private final TezConfiguration amConf;
private final Credentials credentials;
- public AMConfiguration(String queueName, Map<String, String> env,
+ public AMConfiguration(Map<String, String> env,
Map<String, LocalResource> localResources,
TezConfiguration conf, Credentials credentials) {
- this.queueName = queueName;
if (conf != null) {
this.amConf = conf;
} else {
this.amConf = new TezConfiguration();
}
+ this.queueName = this.amConf.get(TezConfiguration.TEZ_QUEUE_NAME);
if (env != null) {
this.env = env;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 7209435..68394b9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -428,7 +428,9 @@ public class TezClientUtils {
appContext.setApplicationType(TezConfiguration.TEZ_APPLICATION_TYPE);
appContext.setApplicationId(appId);
appContext.setResource(capability);
- appContext.setQueue(amConfig.getQueueName());
+ if (amConfig.getQueueName() != null) {
+ appContext.setQueue(amConfig.getQueueName());
+ }
appContext.setApplicationName(amName);
appContext.setCancelTokensWhenComplete(amConfig.getAMConf().getBoolean(
TezConfiguration.TEZ_AM_CANCEL_DELEGATION_TOKEN,
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 59c93a3..6392ded 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -289,4 +289,10 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_PRE_WARM_PB_PLAN_BINARY_PATH =
TEZ_SESSION_PRE_WARM_PREFIX + "dag-plan.pb.path";
+ /**
+ * The queue name for all jobs being submitted as part of a session, or for
+ * non session jobs.
+ */
+ public static final String TEZ_QUEUE_NAME =
+ TEZ_PREFIX + "queue.name";
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index 6067190..bd032e2 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -147,7 +147,7 @@ public class FilterLinesByWord {
- AMConfiguration amConf = new AMConfiguration("default", null, commonLocalResources, tezConf, null);
+ AMConfiguration amConf = new AMConfiguration(null, commonLocalResources, tezConf, null);
TezSessionConfiguration sessionConf = new TezSessionConfiguration(amConf, tezConf);
TezSession tezSession = new TezSession("FilterLinesByWordSession", sessionConf);
tezSession.start(); // Why do I need to start the TezSession.
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index 3c85122..8943dfa 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -827,7 +827,7 @@ public class MRRSleepJob extends Configured implements Tool {
conf.set(TezConfiguration.TEZ_AM_JAVA_OPTS,
MRHelpers.getMRAMJavaOpts(conf));
- AMConfiguration amConfig = new AMConfiguration(null, null,
+ AMConfiguration amConfig = new AMConfiguration(null,
null, conf, null);
DAGClient dagClient =
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
index 1d20592..fa2533b 100644
--- a/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
+++ b/tez-mapreduce-examples/src/main/java/org/apache/tez/mapreduce/examples/OrderedWordCount.java
@@ -336,7 +336,7 @@ public class OrderedWordCount {
// the tez jars.
TezSession tezSession = null;
- AMConfiguration amConfig = new AMConfiguration("default", null,
+ AMConfiguration amConfig = new AMConfiguration(null,
null, tezConf, null);
if (useTezSession) {
LOG.info("Creating Tez Session");
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index 6b6dd35..d5b61b1 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -568,9 +568,10 @@ public class YARNRunner implements ClientProtocol {
try {
dagAMConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
jobSubmitDir);
+ String queueName = jobConf.get(JobContext.QUEUE_NAME,
+ YarnConfiguration.DEFAULT_QUEUE_NAME);
+ dagAMConf.set(TezConfiguration.TEZ_QUEUE_NAME, queueName);
AMConfiguration amConfig = new AMConfiguration(
- jobConf.get(JobContext.QUEUE_NAME,
- YarnConfiguration.DEFAULT_QUEUE_NAME),
environment,
jobLocalResources, dagAMConf, ts);
tezClient.submitDAGApplication(appId, dag, amConfig);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index f0cb289..2a25eac 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -128,6 +128,8 @@ public class DeprecatedKeys {
private static void populateMRToTezRuntimeParamMap() {
+ registerMRToRuntimeKeyTranslation(MRJobConfig.QUEUE_NAME, TezConfiguration.TEZ_QUEUE_NAME);
+
registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD, TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD);
registerMRToRuntimeKeyTranslation(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, TezJobConfig.TEZ_RUNTIME_IFILE_READAHEAD_BYTES);
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/83a657bb/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 1bc7b4d..c2b6451 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -185,7 +185,7 @@ public class TestMRRJobsDAGApi {
new HashMap<String, LocalResource>();
AMConfiguration amConfig = new AMConfiguration(
- "default", commonEnv, amLocalResources,
+ commonEnv, amLocalResources,
tezConf, null);
TezSessionConfiguration tezSessionConfig =
new TezSessionConfiguration(amConfig, tezConf);
@@ -437,7 +437,7 @@ public class TestMRRJobsDAGApi {
boolean reuseSession = reUseTezSession != null;
TezSessionConfiguration tezSessionConfig;
AMConfiguration amConfig = new AMConfiguration(
- "default", commonEnv, amLocalResources,
+ commonEnv, amLocalResources,
tezConf, null);
if(!dagViaRPC) {
// TODO Use utility method post TEZ-205 to figure out AM arguments etc.