You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ry...@apache.org on 2015/02/03 20:56:53 UTC
oozie git commit: OOZIE-2110 cancel delegation token of launcher jobs
that stay till child jobs finish (ryota)
Repository: oozie
Updated Branches:
refs/heads/master d929be9f7 -> e589521a5
OOZIE-2110 cancel delegation token of launcher jobs that stay till child jobs finish (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e589521a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e589521a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e589521a
Branch: refs/heads/master
Commit: e589521a56cf758ae26c6b73010f388e47ad61f1
Parents: d929be9
Author: egashira <ry...@yahoo.com>
Authored: Mon Feb 2 10:08:14 2015 -0800
Committer: egashira <ry...@yahoo.com>
Committed: Mon Feb 2 10:08:14 2015 -0800
----------------------------------------------------------------------
.../org/apache/oozie/action/hadoop/JavaActionExecutor.java | 9 ++++++---
.../apache/oozie/action/hadoop/MapReduceActionExecutor.java | 5 ++++-
.../org/apache/oozie/action/hadoop/ShellActionExecutor.java | 1 -
.../apache/oozie/action/hadoop/TestJavaActionExecutor.java | 4 ++--
release-log.txt | 1 +
.../oozie/action/hadoop/TestMapReduceActionExecutor.java | 5 ++++-
6 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 6cf6ea3..da550a7 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -478,6 +478,9 @@ public class JavaActionExecutor extends ActionExecutor {
setJavaMain(actionConf, actionXml);
parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf);
+
+ // set cancel.delegation.token in actionConf that child job doesn't cancel delegation token
+ actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
return actionConf;
}
catch (IOException ex) {
@@ -821,6 +824,9 @@ public class JavaActionExecutor extends ActionExecutor {
// launcher job configuration
JobConf launcherJobConf = createBaseHadoopConf(context, actionXml);
+ // cancel delegation token on a launcher job which stays alive till child job(s) finishes
+ // otherwise (in mapred action), doesn't cancel not to disturb running child job
+ launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
setupLauncherConf(launcherJobConf, actionXml, appPathRoot, context);
// Properties for when a launcher job's AM gets restarted
@@ -898,9 +904,6 @@ public class JavaActionExecutor extends ActionExecutor {
// maybe we should add queue to the WF schema, below job-tracker
actionConfToLauncherConf(actionConf, launcherJobConf);
- // to disable cancelation of delegation token on launcher job end
- launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
-
return launcherJobConf;
}
catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index 65a4ed2..004adf9 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -87,7 +87,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
@Override
Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
super.setupLauncherConf(conf, actionXml, appPath, context);
- conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
+ conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
return conf;
}
@@ -167,6 +167,9 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
}
}
+ // child job cancel delegation token for mapred action
+ actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
+
return actionConf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
index d1efda1..3be1eaa 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
@@ -117,7 +117,6 @@ public class ShellActionExecutor extends JavaActionExecutor {
protected Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
throws ActionExecutorException {
super.setupLauncherConf(conf, actionXml, appPath, context);
- conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
addDefaultChildEnv(conf);
return conf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index c993132..c18d020 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -288,8 +288,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
conf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf);
assertEquals("LQ", conf.get("mapred.job.queue.name"));
assertEquals("AQ", actionConf.get("mapred.job.queue.name"));
-
-
+ assertEquals(true, conf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", false));
+ assertEquals(false, actionConf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", true));
}
protected Context createContext(String actionXml, String group) throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 3a049a5..c4a2984 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2110 cancel delegation token of launcher jobs that stay till child jobs finish (ryota)
OOZIE-2119 Distcp action fails when -D option in arguments (ryota)
OOZIE-2112 Child Job URL doesn't show properly with Hive on Tez (ryota)
OOZIE-2122 fix test case failure of TestLiteWorkflowAppService (ryota)
http://git-wip-us.apache.org/repos/asf/oozie/blob/e589521a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
index 50927ce..3204c00 100644
--- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
+++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
@@ -229,6 +229,9 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase {
Configuration conf = ae.createBaseHadoopConf(context, actionXml);
ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
assertEquals("IN", conf.get("mapred.input.dir"));
+ JobConf launcherJobConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, conf);
+ assertEquals(false, launcherJobConf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", true));
+ assertEquals(true, conf.getBoolean("mapreduce.job.complete.cancel.delegation.tokens", false));
// Enable uber jars to test that MapReduceActionExecutor picks up the oozie.mapreduce.uber.jar property correctly
Services serv = Services.get();
@@ -239,7 +242,7 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase {
conf = ae.createBaseHadoopConf(context, actionXml);
ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir());
assertEquals(getNameNodeUri() + "/app/job.jar", conf.get("oozie.mapreduce.uber.jar")); // absolute path with namenode
- JobConf launcherJobConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, conf);
+ launcherJobConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, conf);
assertEquals(getNameNodeUri() + "/app/job.jar", launcherJobConf.getJar()); // same for launcher conf
actionXml = createUberJarActionXML("/app/job.jar", "");