You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ry...@apache.org on 2015/02/04 01:19:04 UTC
oozie git commit: OOZIE-2123 Disable launcher uber mode if
classloader options are set (ryota)
Repository: oozie
Updated Branches:
refs/heads/master c6afe1c11 -> 5ab4c2515
OOZIE-2123 Disable launcher uber mode if classloader options are set (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5ab4c251
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5ab4c251
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5ab4c251
Branch: refs/heads/master
Commit: 5ab4c25156e55d98dce03b68bfb6c74ab805195f
Parents: c6afe1c
Author: egashira <ry...@yahoo.com>
Authored: Tue Feb 3 16:17:44 2015 -0800
Committer: egashira <ry...@yahoo.com>
Committed: Tue Feb 3 16:17:44 2015 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 20 ++++++++-
.../action/hadoop/TestJavaActionExecutor.java | 46 ++++++++++++++++++++
release-log.txt | 1 +
3 files changed, 66 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/5ab4c251/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index da550a7..7beac5c 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -114,6 +114,8 @@ public class JavaActionExecutor extends ActionExecutor {
protected XLog LOG = XLog.getLog(getClass());
private static final Pattern heapPattern = Pattern.compile("-Xmx(([0-9]+)[mMgG])");
public static final String CONF_HADOOP_YARN_UBER_MODE = "oozie.action.launcher." + HADOOP_YARN_UBER_MODE;
+ public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader";
+ public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first";
static {
DISALLOWED_PROPERTIES.add(HADOOP_USER);
@@ -897,7 +899,12 @@ public class JavaActionExecutor extends ActionExecutor {
// setting for uber mode
if (launcherJobConf.getBoolean(HADOOP_YARN_UBER_MODE, false)) {
- updateConfForUberMode(launcherJobConf);
+ if (checkPropertiesToDisableUber(launcherJobConf)) {
+ launcherJobConf.setBoolean(HADOOP_YARN_UBER_MODE, false);
+ }
+ else {
+ updateConfForUberMode(launcherJobConf);
+ }
}
// properties from action that are needed by the launcher (e.g. QUEUE NAME, ACLs)
@@ -911,6 +918,17 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
+ private boolean checkPropertiesToDisableUber(Configuration launcherConf) {
+ boolean disable = false;
+ if (launcherConf.getBoolean(HADOOP_JOB_CLASSLOADER, false)) {
+ disable = true;
+ }
+ else if (launcherConf.getBoolean(HADOOP_USER_CLASSPATH_FIRST, false)) {
+ disable = true;
+ }
+ return disable;
+ }
+
private void injectCallback(Context context, Configuration conf) {
String callback = context.getCallbackUrl("$jobStatus");
if (conf.get("job.end.notification.url") != null) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/5ab4c251/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index c18d020..7ee865b 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -1996,6 +1996,52 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals(2560, heapSize);
}
+ public void testDisableUberForProperties() throws Exception {
+ Element actionXml1 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
+ + "<name-node>" + getNameNodeUri() + "</name-node>"
+ + "<configuration>"
+ + "<property><name>oozie.launcher.mapreduce.job.classloader</name>"
+ + "<value>true</value></property>"
+ + "</configuration>"
+ + "<main-class>MAIN-CLASS</main-class>" + "</java>");
+ JavaActionExecutor ae = new JavaActionExecutor();
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+
+ WorkflowJobBean wf = createBaseWorkflow(protoConf, "action");
+ WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
+ action.setType(ae.getType());
+
+ Context context = new Context(wf, action);
+ JobConf launcherConf = new JobConf();
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
+
+ // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true
+ assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
+
+ Element actionXml2 = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>"
+ + "<name-node>" + getNameNodeUri() + "</name-node>"
+ + "<configuration>"
+ + "<property><name>oozie.launcher.mapreduce.user.classpath.first</name>"
+ + "<value>true</value></property>"
+ + "</configuration>"
+ + "<main-class>MAIN-CLASS</main-class>" + "</java>");
+ ae = new JavaActionExecutor();
+ protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+
+ wf = createBaseWorkflow(protoConf, "action");
+ action = (WorkflowActionBean) wf.getActions().get(0);
+ action.setType(ae.getType());
+
+ context = new Context(wf, action);
+ launcherConf = new JobConf();
+ launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml1, launcherConf);
+
+ // uber mode should be disabled since oozie.launcher.mapreduce.job.classloader=true
+ assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_UBER_MODE));
+}
+
public void testAddToCache() throws Exception {
JavaActionExecutor ae = new JavaActionExecutor();
Configuration conf = new XConfiguration();
http://git-wip-us.apache.org/repos/asf/oozie/blob/5ab4c251/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 99b3f7f..2db4bb3 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2123 Disable launcher uber mode if classloader options are set (ryota)
OOZIE-2118 add createdtime option to workflow jobs query (ryota)
OOZIE-2110 cancel delegation token of launcher jobs that stay till child jobs finish (ryota)
OOZIE-2119 Distcp action fails when -D option in arguments (ryota)