You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ry...@apache.org on 2015/06/09 11:13:30 UTC
oozie git commit: OOZIE-2260 Only set java.io.tmpdir to ./tmp for the
AM (ryota)
Repository: oozie
Updated Branches:
refs/heads/master 48b64bc94 -> 999fbcdcb
OOZIE-2260 Only set java.io.tmpdir to ./tmp for the AM (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/999fbcdc
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/999fbcdc
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/999fbcdc
Branch: refs/heads/master
Commit: 999fbcdcb225d0c7b3129c8bf54b3b123f180e1c
Parents: 48b64bc
Author: egashira <ry...@yahoo.com>
Authored: Tue Jun 9 02:10:42 2015 -0700
Committer: egashira <ry...@yahoo.com>
Committed: Tue Jun 9 02:10:42 2015 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 17 -------
.../action/hadoop/TestJavaActionExecutor.java | 52 ++++++++++----------
release-log.txt | 1 +
3 files changed, 27 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/999fbcdc/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 695853e..7a7d4a7 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -407,25 +407,8 @@ public class JavaActionExecutor extends ActionExecutor {
}
void updateConfForJavaTmpDir(Configuration conf) {
- String mapOpts = conf.get(HADOOP_MAP_JAVA_OPTS);
- String reduceOpts = conf.get(HADOOP_REDUCE_JAVA_OPTS);
- String childOpts = conf.get(HADOOP_CHILD_JAVA_OPTS);
String amChildOpts = conf.get(YARN_AM_COMMAND_OPTS);
String oozieJavaTmpDirSetting = "-Djava.io.tmpdir=./tmp";
- if (childOpts == null) {
- conf.set(HADOOP_CHILD_JAVA_OPTS, oozieJavaTmpDirSetting);
- } else {
- conf.set(HADOOP_CHILD_JAVA_OPTS, childOpts + " " + oozieJavaTmpDirSetting);
- }
-
- if (mapOpts != null && !mapOpts.contains(JAVA_TMP_DIR_SETTINGS)) {
- conf.set(HADOOP_MAP_JAVA_OPTS, mapOpts + " " + oozieJavaTmpDirSetting);
- }
-
- if (reduceOpts != null && !reduceOpts.contains(JAVA_TMP_DIR_SETTINGS)) {
- conf.set(HADOOP_REDUCE_JAVA_OPTS, reduceOpts + " " + oozieJavaTmpDirSetting);
- }
-
if (amChildOpts != null && !amChildOpts.contains(JAVA_TMP_DIR_SETTINGS)) {
conf.set(YARN_AM_COMMAND_OPTS, amChildOpts + " " + oozieJavaTmpDirSetting);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/999fbcdc/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index c34020b..1a10352 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -1070,8 +1070,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
Configuration conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf);
- assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts"));
+ assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts"));
+ assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts"));
actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>"
@@ -1095,8 +1095,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf);
- assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts"));
+ assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts"));
+ assertEquals("-Xmx200m JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts"));
actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>"
@@ -1122,8 +1122,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf);
- assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts"));
- assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts"));
+ assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts"));
+ assertEquals("JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts"));
actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>"
@@ -1149,8 +1149,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf);
- assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts"));
+ assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts"));
+ assertEquals("-Xmx200m JAVA-OPT3 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts"));
actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ getNameNodeUri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<job-xml>job2.xml</job-xml>"
@@ -1177,8 +1177,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf);
- assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapred.child.java.opts"));
- assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2 -Djava.io.tmpdir=./tmp", conf.get("mapreduce.map.java.opts"));
+ assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2", conf.get("mapred.child.java.opts"));
+ assertEquals("JAVA-OPT3 JAVA-OPT4 JAVA-OPT1 JAVA-OPT2", conf.get("mapreduce.map.java.opts"));
}
public void testActionLibsPath() throws Exception {
@@ -1749,9 +1749,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
Configuration actionConf = ae.createBaseHadoopConf(context, actionXmlconf);
Configuration conf = ae.createLauncherConf(getFileSystem(), context, action, actionXmlconf, actionConf);
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
conf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS));
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
conf.get(JavaActionExecutor.HADOOP_MAP_JAVA_OPTS));
assertEquals("-Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./usr", conf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS));
assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./usr -Xmx2048m " +
@@ -1778,9 +1778,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
Configuration jobConf = ae2.createBaseHadoopConf(context2, eActionXml);
ae2.setupActionConf(jobConf, context2, eActionXml, appPath2);
- assertEquals("-Xmx200m -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS));
+ assertEquals("-Xmx200m", jobConf.get(JavaActionExecutor.HADOOP_CHILD_JAVA_OPTS));
assertEquals("-Xmx1024m -Djava.io.tmpdir=./usr", jobConf.get(JavaActionExecutor.HADOOP_MAP_JAVA_OPTS));
- assertEquals("-Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS));
+ assertEquals("-Xmx2560m -XX:NewRatio=8", jobConf.get(JavaActionExecutor.HADOOP_REDUCE_JAVA_OPTS));
assertNull(jobConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
}
public void testUpdateConfForUberMode() throws Exception {
@@ -1813,9 +1813,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("2560", launcherConf.get(JavaActionExecutor.YARN_AM_RESOURCE_MB));
// heap size in child.opts (2048 + 512)
int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true",
launcherConf.get("mapreduce.map.java.opts"));
// There's an extra parameter (-Xmx1024m) in here when using YARN that's not here when using MR1
if (createJobConf().get("yarn.resourcemanager.address") != null) {
@@ -1858,8 +1858,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// heap size (2560 + 512)
heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapreduce.map.java.opts"));
+ assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts"));
+ assertEquals("-Xmx1536m -Xmx2560m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts"));
assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx2560m -XX:NewRatio=8 " +
"-Xmx3072m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
assertEquals(3072, heapSize);
@@ -1896,8 +1896,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// heap size (limit to 3584)
heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8 -Djava.io.tmpdir=./tmp", launcherConf.get("mapreduce.map.java.opts"));
+ assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapred.child.java.opts"));
+ assertEquals("-Xmx1536m -Xmx4000m -XX:NewRatio=8", launcherConf.get("mapreduce.map.java.opts"));
assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx1536m -Xmx4000m -XX:NewRatio=8 " +
"-Xmx3584m -Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
assertEquals(3584, heapSize);
@@ -2004,9 +2004,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// heap size (2048 + 512)
int heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2",
launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2",
launcherConf.get("mapreduce.map.java.opts"));
assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Dkey2=val2 -Xmx2560m " +
"-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
@@ -2032,9 +2032,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// heap size (2048 + 512)
heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1",
launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1",
launcherConf.get("mapreduce.map.java.opts"));
assertEquals("-Xmx1024m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx1536m -Xmx2048m -Dkey1=val1 -Xmx2560m " +
"-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
@@ -2060,9 +2060,9 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
// heap size (2048 + 512)
heapSize = ae.extractHeapSizeMB(launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS));
- assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1",
launcherConf.get("mapred.child.java.opts"));
- assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Djava.io.tmpdir=./tmp",
+ assertEquals("-Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1",
launcherConf.get("mapreduce.map.java.opts"));
assertEquals("-Xmx2048m -Djava.net.preferIPv4Stack=true -Xmx200m -Xmx3072m -Xmx1024m -Dkey1=val1 -Xmx2560m " +
"-Djava.io.tmpdir=./tmp", launcherConf.get(JavaActionExecutor.YARN_AM_COMMAND_OPTS).trim());
http://git-wip-us.apache.org/repos/asf/oozie/blob/999fbcdc/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 809020a..78c6494 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -2,6 +2,7 @@
-- Oozie 4.2.0 release (unreleased)
+OOZIE-2260 Only set "java.io.tmpdir" to "./tmp" for the AM (ryota)
OOZIE-2240 add configuration to disable email attachment support (egashira via shwethags)
OOZIE-1963 Create a Hive Server 2 example (qwertymaniac via shwethags)
OOZIE-1993 Rerun fails during join in certain condition (shwethags)