You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2016/12/07 22:59:18 UTC
tez git commit: TEZ-3537. ArrayIndexOutOfBoundsException with empty
environment variables/Port YARN-3768 to Tez (Eric Badger via jeagles)
Repository: tez
Updated Branches:
refs/heads/master a33d2213b -> a2b6eac1c
TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez (Eric Badger via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a2b6eac1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a2b6eac1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a2b6eac1
Branch: refs/heads/master
Commit: a2b6eac1cc4483126afb96e84c355f253dcfbd85
Parents: a33d221
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Wed Dec 7 16:59:04 2016 -0600
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Wed Dec 7 16:59:04 2016 -0600
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../org/apache/tez/common/TezYARNUtils.java | 25 ++++++++-----
.../org/apache/tez/common/TestTezYARNUtils.java | 37 ++++++++++++++++++++
3 files changed, 57 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 6bbbfb2..b15686e 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez
TEZ-3271. Provide mapreduce failures.maxpercent equivalent.
TEZ-3222. Reduce messaging overhead for auto-reduce parallelism case.
TEZ-3547. Add TaskAssignment Analyzer.
@@ -154,6 +155,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez
TEZ-3536. NPE in WebUIService start when host resolution fails.
TEZ-3534. Differentiate thread names on Fetchers, minor changes to shuffle shutdown code.
TEZ-3491. Tez job can hang due to container priority inversion.
@@ -662,6 +664,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez
TEZ-3536. NPE in WebUIService start when host resolution fails.
TEZ-3493. DAG submit timeout cannot be set to a month
TEZ-3505. Move license to the file header for TezBytesWritableSerialization
http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
index 145f928..d5da672 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezYARNUtils.java
@@ -38,8 +38,17 @@ import org.apache.tez.dag.api.TezConstants;
@Private
public class TezYARNUtils {
private static Logger LOG = LoggerFactory.getLogger(TezYARNUtils.class);
-
- private static Pattern ENV_VARIABLE_PATTERN = Pattern.compile(Shell.getEnvironmentVariableRegex());
+
+ public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
+
+ private static final Pattern VAR_SUBBER =
+ Pattern.compile(Shell.getEnvironmentVariableRegex());
+ private static final Pattern VARVAL_SPLITTER = Pattern.compile(
+ "(?<=^|,)" // preceded by ',' or line begin
+ + '(' + ENV_NAME_REGEX + ')' // var group
+ + '='
+ + "([^,]*)" // val group
+ );
public static String getFrameworkClasspath(Configuration conf, boolean usingArchive) {
StringBuilder classpathBuilder = new StringBuilder();
@@ -126,10 +135,10 @@ public class TezYARNUtils {
public static void appendToEnvFromInputString(Map<String, String> env,
String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
- String childEnvs[] = envString.split(",");
- for (String cEnv : childEnvs) {
- String[] parts = cEnv.split("="); // split on '='
- Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+ Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+ while (varValMatcher.find()) {
+ String envVar = varValMatcher.group(1);
+ Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
StringBuffer sb = new StringBuffer();
while (m.find()) {
String var = m.group(1);
@@ -145,7 +154,7 @@ public class TezYARNUtils {
m.appendReplacement(sb, Matcher.quoteReplacement(replace));
}
m.appendTail(sb);
- addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+ addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
}
}
}
@@ -156,7 +165,7 @@ public class TezYARNUtils {
String childEnvs[] = envString.split(",");
for (String cEnv : childEnvs) {
String[] parts = cEnv.split("="); // split on '='
- Matcher m = ENV_VARIABLE_PATTERN.matcher(parts[1]);
+ Matcher m = VAR_SUBBER .matcher(parts[1]);
StringBuffer sb = new StringBuffer();
while (m.find()) {
String var = m.group(1);
http://git-wip-us.apache.org/repos/asf/tez/blob/a2b6eac1/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
index 092bd50..9f02801 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezYARNUtils.java
@@ -19,16 +19,21 @@
package org.apache.tez.common;
import java.io.File;
+import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezConstants;
import org.junit.Assert;
import org.junit.Test;
+import static org.apache.tez.common.TezYARNUtils.appendToEnvFromInputString;
+import static org.junit.Assert.assertEquals;
+
public class TestTezYARNUtils {
@Test(timeout = 5000)
@@ -104,4 +109,36 @@ public class TestTezYARNUtils {
Assert.assertTrue(classpath.indexOf("foobar") >
classpath.indexOf(Environment.PWD.$()));
}
+
+ @Test
+ public void testSetEnvFromInputString() {
+ Map<String, String> environment = new HashMap<String, String>();
+ environment.put("JAVA_HOME", "/path/jdk");
+ String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
+ + ",g_nix=$JAVA_HOME";
+ appendToEnvFromInputString(environment, goodEnv, File.pathSeparator);
+ assertEquals("1", environment.get("a1"));
+ assertEquals("2", environment.get("b_2"));
+ assertEquals("3", environment.get("_c"));
+ assertEquals("4", environment.get("d"));
+ assertEquals("", environment.get("e"));
+ if (Shell.WINDOWS) {
+ assertEquals("$JAVA_HOME", environment.get("g_nix"));
+ assertEquals("/path/jdk", environment.get("f_win"));
+ } else {
+ assertEquals("/path/jdk", environment.get("g_nix"));
+ assertEquals("%JAVA_HOME%", environment.get("f_win"));
+ }
+ String badEnv = "0=,1,,2=a=b,3=a=,4==,5==a,==,c-3=3,=";
+ environment.clear();
+ appendToEnvFromInputString(environment, badEnv, File.pathSeparator);
+ assertEquals(environment.size(), 0);
+
+ // Test "=" in the value part
+ environment.clear();
+ appendToEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2",
+ File.pathSeparator);
+ assertEquals("=", environment.get("e1"));
+ assertEquals("a1=a2", environment.get("e2"));
+ }
}