You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ha...@apache.org on 2016/03/30 16:23:37 UTC
oozie git commit: OOZIE-2481 Add YARN_CONF_DIR in the Shell action
(harsh)
Repository: oozie
Updated Branches:
refs/heads/master ae2c3009a -> a96e2d1c8
OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a96e2d1c
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a96e2d1c
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a96e2d1c
Branch: refs/heads/master
Commit: a96e2d1c856af5cbf09f0702201ae3aa60050a97
Parents: ae2c300
Author: Harsh J <ha...@cloudera.com>
Authored: Wed Mar 30 19:52:50 2016 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Wed Mar 30 19:52:50 2016 +0530
----------------------------------------------------------------------
.../oozie/action/hadoop/TestShellActionExecutor.java | 10 ++++++++--
docs/src/site/twiki/DG_ShellActionExtension.twiki | 4 ++++
release-log.txt | 1 +
.../java/org/apache/oozie/action/hadoop/ShellMain.java | 9 ++++++---
4 files changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
index ca1dee8..6a962a1 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
@@ -60,8 +60,11 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
: "ls -ltr\necho $1 $2\nexit 1";
private static final String PERL_SCRIPT_CONTENT = "print \"MY_VAR=TESTING\";";
private static final String SHELL_SCRIPT_HADOOP_CONF_DIR_CONTENT = Shell.WINDOWS
- ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho HADOOP_CONF_DIR=%HADOOP_CONF_DIR%"
- : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho HADOOP_CONF_DIR=$HADOOP_CONF_DIR";
+ ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho HADOOP_CONF_DIR=%HADOOP_CONF_DIR%\n"
+ : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho HADOOP_CONF_DIR=$HADOOP_CONF_DIR\n";
+ private static final String SHELL_SCRIPT_YARN_CONF_DIR_CONTENT = Shell.WINDOWS
+ ? "echo OOZIE_ACTION_CONF_XML=%OOZIE_ACTION_CONF_XML%\necho YARN_CONF_DIR=%YARN_CONF_DIR%\n"
+ : "echo OOZIE_ACTION_CONF_XML=$OOZIE_ACTION_CONF_XML\necho YARN_CONF_DIR=$YARN_CONF_DIR\n";
/**
* Verify if the ShellActionExecutor indeed setups the basic stuffs
@@ -128,6 +131,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
Path script = new Path(getAppPath(), SHELL_SCRIPTNAME);
Writer w = new OutputStreamWriter(fs.create(script));
w.write(SHELL_SCRIPT_HADOOP_CONF_DIR_CONTENT);
+ w.write(SHELL_SCRIPT_YARN_CONF_DIR_CONTENT);
w.close();
// Create sample Shell action xml
@@ -141,10 +145,12 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
WorkflowAction action = _testSubmit(actionXml, true, "");
String oozieActionConfXml = PropertiesUtils.stringToProperties(action.getData()).getProperty("OOZIE_ACTION_CONF_XML");
String hadoopConfDir = PropertiesUtils.stringToProperties(action.getData()).getProperty("HADOOP_CONF_DIR");
+ String yarnConfDir = PropertiesUtils.stringToProperties(action.getData()).getProperty("YARN_CONF_DIR");
assertNotNull(oozieActionConfXml);
assertNotNull(hadoopConfDir);
String s = new File(oozieActionConfXml).getParent() + File.separator + "oozie-hadoop-conf-";
Assert.assertTrue("Expected HADOOP_CONF_DIR to start with " + s + " but was " + hadoopConfDir, hadoopConfDir.startsWith(s));
+ Assert.assertTrue("Expected YARN_CONF_DIR to start with " + s + " but was " + yarnConfDir, yarnConfDir.startsWith(s));
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/docs/src/site/twiki/DG_ShellActionExtension.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_ShellActionExtension.twiki b/docs/src/site/twiki/DG_ShellActionExtension.twiki
index 4d8a6b1..025a94d 100644
--- a/docs/src/site/twiki/DG_ShellActionExtension.twiki
+++ b/docs/src/site/twiki/DG_ShellActionExtension.twiki
@@ -219,6 +219,10 @@ nameNode=hdfs://localhost:8020
queueName=default
</verbatim>
+---+++ Shell Action Configuration
+
+=oozie.action.shell.setup.hadoop.conf.dir= - Generates a config directory with various core/hdfs/yarn/mapred-site.xml files and points =HADOOP_CONF_DIR= and =YARN_CONF_DIR= env-vars to it, before the Script is invoked. XML is sourced from the action configuration. Useful when the Shell script passed uses various =hadoop= commands. Default is false.
+
---+++ Shell Action Logging
Shell action's stdout and stderr output are redirected to the Oozie Launcher map-reduce job task STDOUT that runs the shell command.
http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index b7402be..3de30b3 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2481 Add YARN_CONF_DIR in the Shell action (harsh)
OOZIE-2492 JSON security issue in js code (fdenes via rkanter)
OOZIE-2429 TestEventGeneration test is flakey (fdenes via rkanter)
OOZIE-2466 Repeated failure of TestMetricsInstrumentation.testSamplers (fdenes via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/a96e2d1c/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
index 553cf1b..e5d8eef 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
@@ -41,6 +41,7 @@ public class ShellMain extends LauncherMain {
public static final String CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR = "oozie.action.shell.setup.hadoop.conf.dir";
public static final String OOZIE_ACTION_CONF_XML = "OOZIE_ACTION_CONF_XML";
private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
+ private static final String YARN_CONF_DIR = "YARN_CONF_DIR";
private static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"};
@@ -117,9 +118,9 @@ public class ShellMain extends LauncherMain {
/**
* This method takes the OOZIE_ACTION_CONF_XML and copies it to Hadoop *-site files in a new directory; it then sets the
- * HADOOP_CONF_DIR to point there. This should allow most Hadoop ecosystem CLI programs to have the proper configuration,
+ * HADOOP/YARN_CONF_DIR to point there. This should allow most Hadoop ecosystem CLI programs to have the proper configuration,
* propagated from Oozie's copy and including anything set in the Workflow's configuration section as well. Otherwise,
- * HADOOP_CONF_DIR points to the NodeManager's *-site files, which are likely not suitable for client programs.
+ * HADOOP/YARN_CONF_DIR points to the NodeManager's *-site files, which are likely not suitable for client programs.
* It will only do this if {@link CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR} is set to true.
*
* @param actionConf The action configuration
@@ -140,8 +141,10 @@ public class ShellMain extends LauncherMain {
dstFiles[i] = new File(confDir, HADOOP_SITE_FILES[i]);
}
copyFileMultiplex(actionXmlFile, dstFiles);
- System.out.println("Setting " + HADOOP_CONF_DIR + " to " + confDir.getAbsolutePath());
+ System.out.println("Setting " + HADOOP_CONF_DIR + " and " + YARN_CONF_DIR
+ + " to " + confDir.getAbsolutePath());
envp.put(HADOOP_CONF_DIR, confDir.getAbsolutePath());
+ envp.put(YARN_CONF_DIR, confDir.getAbsolutePath());
}
}
}