You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ro...@apache.org on 2016/05/27 19:32:29 UTC
tez git commit: TEZ-909. Provide support for application tags
(ebadger via rohini)
Repository: tez
Updated Branches:
refs/heads/master 89802b1c5 -> de7fd9aa5
TEZ-909. Provide support for application tags (ebadger via rohini)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/de7fd9aa
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/de7fd9aa
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/de7fd9aa
Branch: refs/heads/master
Commit: de7fd9aa5dea40a3cd62fd8bee3c2bf96065a9fe
Parents: 89802b1
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Fri May 27 12:32:21 2016 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Fri May 27 12:32:21 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/client/TezClientUtils.java | 7 +++++++
.../org/apache/tez/dag/api/TezConfiguration.java | 8 ++++++++
.../org/apache/tez/client/TestTezClientUtils.java | 17 +++++++++++++++++
.../tez/mapreduce/hadoop/DeprecatedKeys.java | 2 ++
.../apache/tez/mapreduce/hadoop/MRJobConfig.java | 2 ++
6 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 07052e5..ed41b07 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-909. Provide support for application tags
TEZ-3257. Fix flaky test TestUnorderedPartitionedKVWriter.
TEZ-3206. Have unordered partitioned KV output send partition stats via VertexManagerEvent.
TEZ-3237. Corrupted shuffle transfers to disk are not detected during transfer
http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index a1ad1d8..eb1a95e 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -704,7 +704,14 @@ public class TezClientUtils {
ApplicationSubmissionContext appContext = Records
.newRecord(ApplicationSubmissionContext.class);
+ Collection<String> tagsFromConf =
+ amConfig.getTezConfiguration().getTrimmedStringCollection(
+ TezConfiguration.TEZ_APPLICATION_TAGS);
+
appContext.setApplicationType(TezConstants.TEZ_APPLICATION_TYPE);
+ if (tagsFromConf != null && !tagsFromConf.isEmpty()) {
+ appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
+ }
appContext.setApplicationId(appId);
appContext.setResource(capability);
if (amConfig.getQueueName() != null) {
http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 9f9defe..4118bb5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -1141,6 +1141,14 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_QUEUE_NAME = TEZ_PREFIX + "queue.name";
/**
+ * String value. Tags for the job that will be passed to YARN at submission
+ * time. Queries to YARN for applications can filter on these tags.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_APPLICATION_TAGS = TEZ_PREFIX + "application.tags";
+
+ /**
* String value. The job priority of submitted application.
*/
@ConfigurationScope(Scope.AM)
http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 4948260..2c69d77 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -32,7 +32,9 @@ import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -328,6 +330,21 @@ public class TestTezClientUtils {
assertNull(appContext.getPriority());
}
+ @Test(timeout=1000)
+ public void testSetApplicationTags() {
+ TezConfiguration conf = new TezConfiguration(false);
+ conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar");
+ AMConfiguration amconfig = new AMConfiguration(conf, null, null);
+ ApplicationSubmissionContext appContext = Records
+ .newRecord(ApplicationSubmissionContext.class);
+ Collection<String> tagsFromConf =
+ amconfig.getTezConfiguration().getTrimmedStringCollection(
+ TezConfiguration.TEZ_APPLICATION_TAGS);
+ appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
+ assertTrue(appContext.getApplicationTags().contains("foo"));
+ assertTrue(appContext.getApplicationTags().contains("bar"));
+ }
+
@Test(timeout = 5000)
public void testSessionTokenInAmClc() throws IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 1335aa2..a826f6d 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -84,6 +84,8 @@ public class DeprecatedKeys {
TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
mrParamToDAGParamMap.put(MRJobConfig.TASK_TIMEOUT,
TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS);
+ mrParamToDAGParamMap.put(MRJobConfig.JOB_TAGS,
+ TezConfiguration.TEZ_APPLICATION_TAGS);
}
// TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild.
http://git-wip-us.apache.org/repos/asf/tez/blob/de7fd9aa/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
index d44be30..7db98bc 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRJobConfig.java
@@ -64,6 +64,8 @@ public interface MRJobConfig {
public static final String QUEUE_NAME = "mapreduce.job.queuename";
+ public static final String JOB_TAGS = "mapreduce.job.tags";
+
public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks";
public static final String SPLIT_FILE = "mapreduce.job.splitfile";