You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2015/04/28 22:40:45 UTC
[04/37] tez git commit: TEZ-2364. Resolve config parameter
replacement on the client, before sending them to the AM. (sseth)
TEZ-2364. Resolve config parameter replacement on the client, before
sending them to the AM. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/2935ef41
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/2935ef41
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/2935ef41
Branch: refs/heads/TEZ-2003
Commit: 2935ef415642a9869166a1d5e1484a635e4401a2
Parents: bbbebb8
Author: Siddharth Seth <ss...@apache.org>
Authored: Fri Apr 24 13:42:01 2015 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Fri Apr 24 13:42:01 2015 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../org/apache/tez/client/TezClientUtils.java | 2 +-
.../apache/tez/dag/api/TezConfiguration.java | 2 +-
.../apache/tez/client/TestTezClientUtils.java | 36 ++++++++++++++++----
4 files changed, 33 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e21d178..92c1f6e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
ALL CHANGES:
+ TEZ-2364. Resolve config parameter replacement on the client, before sending them to the AM.
TEZ-2298. Avoid logging full exception trace in TaskRunner when it's not the main error reason and is ignored.
TEZ-2248. VertexImpl/DAGImpl.checkForCompletion have too many termination cause checks
TEZ-2341. TestMockDAGAppMaster.testBasicCounters fails on windows
http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index d12ece2..86ba0f9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -743,7 +743,7 @@ public class TezClientUtils {
for (Entry<String, String> entry : amConf) {
PlanKeyValuePair.Builder kvp = PlanKeyValuePair.newBuilder();
kvp.setKey(entry.getKey());
- kvp.setValue(entry.getValue());
+ kvp.setValue(amConf.get(entry.getKey()));
builder.addConfKeyValues(kvp);
}
if (additionalConfigs != null && !additionalConfigs.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 7a49ee5..50b3e00 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -133,7 +133,7 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_STAGING_DIR = TEZ_PREFIX + "staging-dir";
public static final String TEZ_AM_STAGING_DIR_DEFAULT = "/tmp/"
+ System.getProperty("user.name") + "/tez/staging";
-
+
/**
* String value that is a file path.
* Path to a credentials file (with serialized credentials) located on the local file system.
http://git-wip-us.apache.org/repos/asf/tez/blob/2935ef41/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
index ea73ab3..cfba917 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClientUtils.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.DataInputByteBuffer;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -68,7 +67,8 @@ import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.Vertex;
-import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
+import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
import org.junit.Assert;
import org.junit.Test;
@@ -283,7 +283,8 @@ public class TestTezClientUtils {
public void testAMLoggingOptsPerLogger() throws IOException, YarnException {
TezConfiguration tezConf = new TezConfiguration();
- tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL, "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
+ tezConf.set(TezConfiguration.TEZ_AM_LOG_LEVEL,
+ "WARN;org.apache.hadoop.ipc=DEBUG;org.apache.hadoop.security=DEBUG");
ApplicationId appId = ApplicationId.newInstance(1000, 1);
Credentials credentials = new Credentials();
@@ -301,7 +302,8 @@ public class TestTezClientUtils {
List<String> expectedCommands = new LinkedList<String>();
expectedCommands.add("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator");
- expectedCommands.add("-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
+ expectedCommands.add(
+ "-Dlog4j.configuration=" + TezConstants.TEZ_CONTAINER_LOG4J_PROPERTIES_FILE);
expectedCommands.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
ApplicationConstants.LOG_DIR_EXPANSION_VAR);
expectedCommands.add("-D" + TezConstants.TEZ_ROOT_LOGGER_NAME + "=" + "WARN" + "," +
@@ -465,6 +467,28 @@ public class TestTezClientUtils {
&& javaOpts.contains("-Dlog4j.configuratorClass=org.apache.tez.common.TezLog4jConfigurator"));
}
+ @Test (timeout = 5000)
+ public void testConfSerializationForAm() {
+ Configuration conf =new Configuration(false);
+ String val1 = "fixedProperty";
+ String val2 = "parametrizedProperty/${user.name}";
+ String expVal2 = "parametrizedProperty/" + System.getProperty("user.name");
+ conf.set("property1", val1);
+ conf.set("property2", val2);
+
+ Map<String, String> expected = new HashMap<String, String>();
+ expected.put("property1", val1);
+ expected.put("property2", expVal2);
+
+ ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(conf, null);
+
+ for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
+ String v = expected.remove(kvPair.getKey());
+ assertEquals(v, kvPair.getValue());
+ }
+ assertTrue(expected.isEmpty());
+ }
+
// To run this test case see TestTezCommonUtils::testLocalResourceVisibility
// We do not have much control over the directory structure, cannot mock as the functions are
// static and do not want to spin up a minitez cluster just for this.
@@ -560,9 +584,9 @@ public class TestTezClientUtils {
srcConf.set(entry.getKey(), entry.getValue());
}
- DAGProtos.ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf);
+ ConfigurationProto confProto = TezClientUtils.createFinalConfProtoForApp(srcConf);
- for (DAGProtos.PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
+ for (PlanKeyValuePair kvPair : confProto.getConfKeyValuesList()) {
String val = confMap.remove(kvPair.getKey());
assertNotNull(val);
assertEquals(val, kvPair.getValue());