You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/09/11 09:20:27 UTC
git commit: TEZ-1534. Make client side configs available to AM and
tasks. (sseth)
Repository: tez
Updated Branches:
refs/heads/master 43e47bfac -> 90ef95ad6
TEZ-1534. Make client side configs available to AM and tasks. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/90ef95ad
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/90ef95ad
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/90ef95ad
Branch: refs/heads/master
Commit: 90ef95ad6742fb12de43bcfbb10c3a74af15233c
Parents: 43e47bf
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Sep 11 00:19:41 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Sep 11 00:19:41 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/client/TezClientUtils.java | 48 ++++++--------------
.../apache/tez/dag/api/TezConfiguration.java | 1 -
.../apache/tez/client/TestTezClientUtils.java | 27 +++++++++++
4 files changed, 41 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/90ef95ad/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0c0bb39..032438f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -21,6 +21,7 @@ ALL CHANGES
TEZ-1447. Provide a mechanism for InputInitializers to know about Vertex state changes.
TEZ-1362. Remove DAG_COMPLETED in DAGEventType.
TEZ-1519. TezTaskRunner should not initialize TezConfiguration in TezChild.
+ TEZ-1534. Make client side configs available to AM and tasks.
Release 0.5.0: Unreleased
http://git-wip-us.apache.org/repos/asf/tez/blob/90ef95ad/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index b53f00c..adda0a9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -375,7 +375,6 @@ public class TezClientUtils {
/**
* Create an ApplicationSubmissionContext to launch a Tez AM
- * @param conf TezConfiguration
* @param appId Application Id
* @param dag DAG to be submitted
* @param amName Name for the application
@@ -501,23 +500,12 @@ public class TezClientUtils {
amLocalResources.putAll(tezJarResources);
// emit conf as PB file
- Configuration finalTezConf = createFinalTezConfForApp(amConfig.getTezConfiguration());
+ ConfigurationProto finalConfProto = createFinalConfProtoForApp(amConfig.getTezConfiguration());
FSDataOutputStream amConfPBOutBinaryStream = null;
try {
- ConfigurationProto.Builder confProtoBuilder =
- ConfigurationProto.newBuilder();
- Iterator<Entry<String, String>> iter = finalTezConf.iterator();
- while (iter.hasNext()) {
- Entry<String, String> entry = iter.next();
- PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
- kvp.setKey(entry.getKey());
- kvp.setValue(entry.getValue());
- confProtoBuilder.addConfKeyValues(kvp);
- }
- //binary output
amConfPBOutBinaryStream = TezCommonUtils.createFileForAM(fs, binaryConfPath);
- confProtoBuilder.build().writeTo(amConfPBOutBinaryStream);
+ finalConfProto.writeTo(amConfPBOutBinaryStream);
} finally {
if(amConfPBOutBinaryStream != null){
amConfPBOutBinaryStream.close();
@@ -560,7 +548,7 @@ public class TezClientUtils {
sessionJarsPBLRsrc);
String user = UserGroupInformation.getCurrentUser().getShortUserName();
- ACLManager aclManager = new ACLManager(user, finalTezConf);
+ ACLManager aclManager = new ACLManager(user, amConfig.getTezConfiguration());
Map<ApplicationAccessType, String> acls = aclManager.toYARNACls();
if(dag != null) {
@@ -625,7 +613,7 @@ public class TezClientUtils {
appContext.setAMContainerSpec(amContainer);
appContext.setMaxAppAttempts(
- finalTezConf.getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
+ amConfig.getTezConfiguration().getInt(TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS,
TezConfiguration.TEZ_AM_MAX_APP_ATTEMPTS_DEFAULT));
return appContext;
@@ -703,26 +691,16 @@ public class TezClientUtils {
+ "," + TezConstants.TEZ_CONTAINER_LOGGER_NAME);
}
- static Configuration createFinalTezConfForApp(TezConfiguration amConf) {
- Configuration conf = new Configuration(false);
- conf.setQuietMode(true);
-
+ static ConfigurationProto createFinalConfProtoForApp(Configuration amConf) {
assert amConf != null;
-
- Entry<String, String> entry;
- Iterator<Entry<String, String>> iter = amConf.iterator();
- while (iter.hasNext()) {
- entry = iter.next();
- // Copy all tez config parameters.
- if (entry.getKey().startsWith(TezConfiguration.TEZ_PREFIX)) {
- conf.set(entry.getKey(), entry.getValue());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding tez dag am parameter from amConf: " + entry.getKey()
- + ", with value: " + entry.getValue());
- }
- }
- }
- return conf;
+ ConfigurationProto.Builder builder = ConfigurationProto.newBuilder();
+ for (Entry<String, String> entry : amConf) {
+ PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
+ kvp.setKey(entry.getKey());
+ kvp.setValue(entry.getValue());
+ builder.addConfKeyValues(kvp);
+ }
+ return builder.build();
}
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/90ef95ad/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 8053283..21b9140 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -723,7 +723,6 @@ public class TezConfiguration extends Configuration {
public static final int TEZ_AM_INLINE_TASK_EXECUTION_MAX_TASKS_DEFAULT = 1;
-
// ACLs related configuration
// Format supports a comma-separated list of users and groups with the users and groups separated
// by whitespace. e.g. "user1,user2 group1,group2"
http://git-wip-us.apache.org/repos/asf/tez/blob/90ef95ad/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index 04c926e..ddd5963 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -17,13 +17,16 @@
*/
package org.apache.tez.client;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileNotFoundException;
import java.net.URL;
import java.net.URLClassLoader;
+import java.util.HashMap;
import java.util.Map;
import java.util.Set;
@@ -39,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
+import org.apache.tez.dag.api.records.DAGProtos;
import org.junit.Assert;
import org.junit.Test;
@@ -301,4 +305,27 @@ public class TestTezClientUtils {
}
}
+ @Test(timeout = 5000)
+ public void testConfigurationAllowAll() {
+ Configuration srcConf = new Configuration(false);
+
+ Map<String, String> confMap = new HashMap<String, String>();
+ confMap.put("ipc.timeout", "2000");
+ confMap.put("fs.defaultFS", "testfs:///");
+ confMap.put("tez.property", "tezProperty");
+ confMap.put("yarn.property", "yarnProperty");
+
+ for (Map.Entry<String, String> entry : confMap.entrySet()) {
+ srcConf.set(entry.getKey(), entry.getValue());
+ }
+
+ DAGProtos.ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf);
+
+ for (DAGProtos.PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
+ String val = confMap.remove(kvPair.getKey());
+ assertNotNull(val);
+ assertEquals(val, kvPair.getValue());
+ }
+ assertTrue(confMap.isEmpty());
+ }
}