You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2016/10/10 11:52:48 UTC
[49/50] [abbrv] oozie git commit: OOZIE-2598 refactoring classes so
that Shell action doesn't need MR jars
OOZIE-2598 refactoring classes so that Shell action doesn't need MR jars
Change-Id: I48f5abe8a8beb68ea96fdb1d72b6601e561074ea
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/165847cc
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/165847cc
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/165847cc
Branch: refs/heads/oya
Commit: 165847ccbc0b1ee0bc084648173de561fe519259
Parents: b1069b2
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Tue Oct 4 15:02:53 2016 +0200
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Tue Oct 4 15:02:53 2016 +0200
----------------------------------------------------------------------
.../action/hadoop/Hive2ActionExecutor.java | 4 +-
.../oozie/action/hadoop/HiveActionExecutor.java | 4 +-
.../oozie/action/hadoop/JavaActionExecutor.java | 1 +
.../action/hadoop/MapReduceActionExecutor.java | 4 +-
.../oozie/action/hadoop/PigActionExecutor.java | 4 +-
.../action/hadoop/ShellActionExecutor.java | 2 +-
.../action/hadoop/SqoopActionExecutor.java | 2 +-
.../wf/SubmitScriptLanguageXCommand.java | 6 +--
.../oozie/command/wf/SubmitSqoopXCommand.java | 4 +-
.../oozie/action/hadoop/TestShellMain.java | 5 +-
.../command/wf/TestSubmitHiveXCommand.java | 6 +--
.../oozie/command/wf/TestSubmitPigXCommand.java | 8 ++--
.../command/wf/TestSubmitSqoopXCommand.java | 4 +-
.../apache/oozie/action/hadoop/HiveMain.java | 4 +-
.../oozie/action/hadoop/TestHiveMain.java | 4 +-
.../apache/oozie/action/hadoop/Hive2Main.java | 4 +-
.../apache/oozie/action/hadoop/ActionUtils.java | 49 ++++++++++++++++++++
.../apache/oozie/action/hadoop/LauncherAM.java | 6 ---
.../oozie/action/hadoop/MapReduceMain.java | 29 ++++--------
.../apache/oozie/action/hadoop/ShellMain.java | 8 ++--
.../org/apache/oozie/action/hadoop/PigMain.java | 4 +-
.../oozie/action/hadoop/PigMainWithOldAPI.java | 4 +-
.../action/hadoop/TestPigMainWithOldAPI.java | 5 +-
.../apache/oozie/action/hadoop/SqoopMain.java | 2 +-
.../oozie/action/hadoop/StreamingMain.java | 4 +-
25 files changed, 104 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
index ddb1504..35277ae 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
@@ -106,7 +106,7 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor {
for (int i = 0; i < params.size(); i++) {
strParams[i] = params.get(i).getTextTrim();
}
- MapReduceMain.setStrings(conf, HIVE2_PARAMS, strParams);
+ ActionUtils.setStrings(conf, HIVE2_PARAMS, strParams);
String[] strArgs = null;
List<Element> eArgs = actionXml.getChildren("argument", ns);
@@ -116,7 +116,7 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor {
strArgs[i] = eArgs.get(i).getTextTrim();
}
}
- MapReduceMain.setStrings(conf, HIVE2_ARGS, strArgs);
+ ActionUtils.setStrings(conf, HIVE2_ARGS, strArgs);
return conf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
index d05edb8..82a955c 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
@@ -98,7 +98,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor {
for (int i = 0; i < params.size(); i++) {
strParams[i] = params.get(i).getTextTrim();
}
- MapReduceMain.setStrings(conf, HIVE_PARAMS, strParams);
+ ActionUtils.setStrings(conf, HIVE_PARAMS, strParams);
String[] strArgs = null;
List<Element> eArgs = actionXml.getChildren("argument", ns);
@@ -108,7 +108,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor {
strArgs[i] = eArgs.get(i).getTextTrim();
}
}
- MapReduceMain.setStrings(conf, HIVE_ARGS, strArgs);
+ ActionUtils.setStrings(conf, HIVE_ARGS, strArgs);
return conf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 0515d39..284690b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -1169,6 +1169,7 @@ public class JavaActionExecutor extends ActionExecutor {
// FIXME: move this to specific places where it's actually needed - keeping it here for now
ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf);
+
// FIXME: Pyspark fix
// FIXME: Do we want to support mapred.child.env?
env.put("SPARK_HOME", ".");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index 89f30f1..f7d230f 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -309,8 +309,8 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
if (recordReader != null) {
conf.set("oozie.streaming.record-reader", recordReader);
}
- MapReduceMain.setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping);
- MapReduceMain.setStrings(conf, "oozie.streaming.env", env);
+ ActionUtils.setStrings(conf, "oozie.streaming.record-reader-mapping", recordReaderMapping);
+ ActionUtils.setStrings(conf, "oozie.streaming.env", env);
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
index 65e9cbf..cbb7436 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
@@ -98,8 +98,8 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor {
public static void setPigScript(Configuration conf, String script, String[] params, String[] args) {
conf.set(PIG_SCRIPT, script);
- MapReduceMain.setStrings(conf, PIG_PARAMS, params);
- MapReduceMain.setStrings(conf, PIG_ARGS, args);
+ ActionUtils.setStrings(conf, PIG_PARAMS, params);
+ ActionUtils.setStrings(conf, PIG_ARGS, args);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
index 5acd581..9d71b78 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
@@ -112,7 +112,7 @@ public class ShellActionExecutor extends JavaActionExecutor {
}
}
}
- MapReduceMain.setStrings(actionConf, key, strTagValue);
+ ActionUtils.setStrings(actionConf, key, strTagValue);
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
index b0fc83a..c6266df 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
@@ -108,7 +108,7 @@ public class SqoopActionExecutor extends JavaActionExecutor {
}
private void setSqoopCommand(Configuration conf, String[] args) {
- MapReduceMain.setStrings(conf, SQOOP_ARGS, args);
+ ActionUtils.setStrings(conf, SQOOP_ARGS, args);
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
index b82369c..fab4398 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitScriptLanguageXCommand.java
@@ -19,7 +19,7 @@
package org.apache.oozie.command.wf;
import org.apache.hadoop.conf.Configuration;
-import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.apache.oozie.action.hadoop.ActionUtils;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.command.CommandException;
import org.jdom.Element;
@@ -58,7 +58,7 @@ public abstract class SubmitScriptLanguageXCommand extends SubmitHttpXCommand {
List<String> Dargs = new ArrayList<String>();
List<String> otherArgs = new ArrayList<String>();
- String[] args = MapReduceMain.getStrings(conf, getOptions());
+ String[] args = ActionUtils.getStrings(conf, getOptions());
for (String arg : args) {
if (arg.startsWith("-D")) {
Dargs.add(arg);
@@ -67,7 +67,7 @@ public abstract class SubmitScriptLanguageXCommand extends SubmitHttpXCommand {
otherArgs.add(arg);
}
}
- String [] params = MapReduceMain.getStrings(conf, getScriptParamters());
+ String [] params = ActionUtils.getStrings(conf, getScriptParamters());
// configuration section
if (Dargs.size() > 0) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
index 51f3d03..c5574c5 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitSqoopXCommand.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.command.CommandException;
-import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.apache.oozie.action.hadoop.ActionUtils;
import org.jdom.Namespace;
import org.jdom.Element;
@@ -57,7 +57,7 @@ public class SubmitSqoopXCommand extends SubmitHttpXCommand {
ele.addContent(nn);
List<String> Dargs = new ArrayList<String>();
- String[] args = MapReduceMain.getStrings(conf, getOptions());
+ String[] args = ActionUtils.getStrings(conf, getOptions());
for (String arg : args) {
if (arg.startsWith("-D")) {
Dargs.add(arg);
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java
index e757e54..a7d6c18 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellMain.java
@@ -25,7 +25,6 @@ import java.io.FileWriter;
import java.io.Writer;
import java.util.Properties;
-import org.apache.hadoop.fs.Path;
import org.apache.oozie.util.XConfiguration;
//Test cases are mainly implemented in the Base class
@@ -53,8 +52,8 @@ public class TestShellMain extends ShellTestCase {
jobConf.set(ShellMain.CONF_OOZIE_SHELL_EXEC, SHELL_COMMAND_NAME);
String[] args = new String[] { SHELL_COMMAND_SCRIPTFILE_OPTION, script.toString(), "A", "B" };
- MapReduceMain.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ARGS, args);
- MapReduceMain.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ENVS,
+ ActionUtils.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ARGS, args);
+ ActionUtils.setStrings(jobConf, ShellMain.CONF_OOZIE_SHELL_ENVS,
new String[] { "var1=value1", "var2=value2" });
File actionXml = new File(getTestCaseDir(), "action.xml");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
index 014daff..98c94a7 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitHiveXCommand.java
@@ -21,7 +21,7 @@ package org.apache.oozie.command.wf;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.local.LocalOozie;
-import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.apache.oozie.action.hadoop.ActionUtils;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.XLog;
@@ -54,9 +54,9 @@ public class TestSubmitHiveXCommand extends XFsTestCase {
String hiveArgsStr = "-a aaa -b bbb -c ccc -M -Da=aaa -Db=bbb -param input=abc";
String[] args = hiveArgsStr.split(" ");
- MapReduceMain.setStrings(conf, XOozieClient.HIVE_OPTIONS, args);
+ ActionUtils.setStrings(conf, XOozieClient.HIVE_OPTIONS, args);
String[] params = new String[]{"INPUT=/some/path", "OUTPUT=/some/other/path", "abc=xyz"};
- MapReduceMain.setStrings(conf, XOozieClient.HIVE_SCRIPT_PARAMS, params);
+ ActionUtils.setStrings(conf, XOozieClient.HIVE_SCRIPT_PARAMS, params);
SubmitHiveXCommand submitHiveCmd = new SubmitHiveXCommand(conf);
String xml = submitHiveCmd.getWorkflowXml(conf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
index e3ce56b..c3cd1aa 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitPigXCommand.java
@@ -21,7 +21,7 @@ package org.apache.oozie.command.wf;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.local.LocalOozie;
-import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.apache.oozie.action.hadoop.ActionUtils;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.XLog;
@@ -55,9 +55,9 @@ public class TestSubmitPigXCommand extends XFsTestCase {
String pigArgsStr = "-a aaa -b bbb -c ccc -M -Da=aaa -Db=bbb -param input=abc";
String[] args = pigArgsStr.split(" ");
- MapReduceMain.setStrings(conf, XOozieClient.PIG_OPTIONS, args);
+ ActionUtils.setStrings(conf, XOozieClient.PIG_OPTIONS, args);
String[] params = new String[]{"INPUT=/some/path", "OUTPUT=/some/other/path", "abc=xyz"};
- MapReduceMain.setStrings(conf, XOozieClient.PIG_SCRIPT_PARAMS, params);
+ ActionUtils.setStrings(conf, XOozieClient.PIG_SCRIPT_PARAMS, params);
SubmitPigXCommand submitPigCmd = new SubmitPigXCommand(conf);
String xml = submitPigCmd.getWorkflowXml(conf);
@@ -128,7 +128,7 @@ public class TestSubmitPigXCommand extends XFsTestCase {
String[] args = new String[2];
args[0] = "-a";
args[1] = "aaa bbb";
- MapReduceMain.setStrings(conf, XOozieClient.PIG_OPTIONS, args);
+ ActionUtils.setStrings(conf, XOozieClient.PIG_OPTIONS, args);
SubmitPigXCommand submitPigCmd = new SubmitPigXCommand(conf);
String xml = submitPigCmd.getWorkflowXml(conf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
index 05fef6a..f2f248a 100644
--- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitSqoopXCommand.java
@@ -21,7 +21,7 @@ package org.apache.oozie.command.wf;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.local.LocalOozie;
-import org.apache.oozie.action.hadoop.MapReduceMain;
+import org.apache.oozie.action.hadoop.ActionUtils;
import org.apache.oozie.client.XOozieClient;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.XLog;
@@ -54,7 +54,7 @@ public class TestSubmitSqoopXCommand extends XFsTestCase {
String sqoopArgsStr = "-Da=aaa -Db=bbb";
String[] args = sqoopArgsStr.split(" ");
- MapReduceMain.setStrings(conf, XOozieClient.SQOOP_OPTIONS, args);
+ ActionUtils.setStrings(conf, XOozieClient.SQOOP_OPTIONS, args);
SubmitSqoopXCommand submitSqoopCmd = new SubmitSqoopXCommand(conf);
String xml = submitSqoopCmd.getWorkflowXml(conf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index 1cf44db..f59750b 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -272,7 +272,7 @@ public class HiveMain extends LauncherMain {
}
// Pass any parameters to Hive via arguments
- String[] params = MapReduceMain.getStrings(hiveConf, HiveActionExecutor.HIVE_PARAMS);
+ String[] params = ActionUtils.getStrings(hiveConf, HiveActionExecutor.HIVE_PARAMS);
if (params.length > 0) {
System.out.println("Parameters:");
System.out.println("------------------------");
@@ -292,7 +292,7 @@ public class HiveMain extends LauncherMain {
System.out.println();
}
- String[] hiveArgs = MapReduceMain.getStrings(hiveConf, HiveActionExecutor.HIVE_ARGS);
+ String[] hiveArgs = ActionUtils.getStrings(hiveConf, HiveActionExecutor.HIVE_ARGS);
for (String hiveArg : hiveArgs) {
if (DISALLOWED_HIVE_OPTIONS.contains(hiveArg)) {
throw new RuntimeException("Error: Hive argument " + hiveArg + " is not supported");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
index 879ae2d..b724db3 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
@@ -102,10 +102,10 @@ public class TestHiveMain extends MainTestCase {
SharelibUtils.addToDistributedCache("hive", fs, getFsTestCaseDir(), jobConf);
jobConf.set(HiveActionExecutor.HIVE_SCRIPT, script.toString());
- MapReduceMain.setStrings(jobConf, HiveActionExecutor.HIVE_PARAMS, new String[]{
+ ActionUtils.setStrings(jobConf, HiveActionExecutor.HIVE_PARAMS, new String[]{
"IN=" + inputDir.toUri().getPath(),
"OUT=" + outputDir.toUri().getPath()});
- MapReduceMain.setStrings(jobConf, HiveActionExecutor.HIVE_ARGS,
+ ActionUtils.setStrings(jobConf, HiveActionExecutor.HIVE_ARGS,
new String[]{ "-v" });
File actionXml = new File(getTestCaseDir(), "action.xml");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index 157fbef..ccf2aff 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -179,7 +179,7 @@ public class Hive2Main extends LauncherMain {
}
// Pass any parameters to Beeline via arguments
- String[] params = MapReduceMain.getStrings(actionConf, Hive2ActionExecutor.HIVE2_PARAMS);
+ String[] params = ActionUtils.getStrings(actionConf, Hive2ActionExecutor.HIVE2_PARAMS);
if (params.length > 0) {
System.out.println("Parameters:");
System.out.println("------------------------");
@@ -204,7 +204,7 @@ public class Hive2Main extends LauncherMain {
arguments.add("-a");
arguments.add("delegationToken");
- String[] beelineArgs = MapReduceMain.getStrings(actionConf, Hive2ActionExecutor.HIVE2_ARGS);
+ String[] beelineArgs = ActionUtils.getStrings(actionConf, Hive2ActionExecutor.HIVE2_ARGS);
for (String beelineArg : beelineArgs) {
if (DISALLOWED_BEELINE_OPTIONS.contains(beelineArg)) {
throw new RuntimeException("Error: Beeline argument " + beelineArg + " is not supported");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java
new file mode 100644
index 0000000..3002ad5
--- /dev/null
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ActionUtils.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.conf.Configuration;
+
+public final class ActionUtils {
+
+ private ActionUtils() {
+ // no instances
+ }
+
+ public static void setStrings(Configuration conf, String key, String[] values) {
+ if (values != null) {
+ conf.setInt(key + ".size", values.length);
+ for (int i = 0; i < values.length; i++) {
+ conf.set(key + "." + i, values[i]);
+ }
+ }
+ }
+
+ public static String[] getStrings(Configuration conf, String key) {
+ String[] values = new String[conf.getInt(key + ".size", 0)];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = conf.get(key + "." + i);
+ if (values[i] == null) {
+ values[i] = "";
+ }
+ }
+ return values;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
index d51dcf7..85d78c6 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
@@ -40,18 +40,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
-import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.exceptions.YarnException;
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index b808535..16cf4b1 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -123,31 +123,20 @@ public class MapReduceMain extends LauncherMain {
return runJob;
}
- protected JobClient createJobClient(JobConf jobConf) throws IOException {
- return new JobClient(jobConf);
- }
-
- // allows any character in the value, the conf.setStrings() does not allow
- // commas
+ @Deprecated
public static void setStrings(Configuration conf, String key, String[] values) {
- if (values != null) {
- conf.setInt(key + ".size", values.length);
- for (int i = 0; i < values.length; i++) {
- conf.set(key + "." + i, values[i]);
- }
- }
+ ActionUtils.setStrings(conf, key, values);
}
+ @Deprecated
public static String[] getStrings(Configuration conf, String key) {
- String[] values = new String[conf.getInt(key + ".size", 0)];
- for (int i = 0; i < values.length; i++) {
- values[i] = conf.get(key + "." + i);
- if (values[i] == null) {
- values[i] = "";
- }
- }
- return values;
+ return ActionUtils.getStrings(conf, key);
+ }
+
+ protected JobClient createJobClient(JobConf jobConf) throws IOException {
+ return new JobClient(jobConf);
}
+
/**
* Will run the user specified OozieActionConfigurator subclass (if one is provided) to update the action configuration.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
index f109318..0ee35e8 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
@@ -24,10 +24,10 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
-import java.io.PrintWriter;
-import java.io.StringReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -186,7 +186,7 @@ public class ShellMain extends LauncherMain {
*/
private Map<String, String> getEnvMap(Map<String, String> envp, Configuration actionConf) {
// Adding user-specified environments
- String[] envs = MapReduceMain.getStrings(actionConf, CONF_OOZIE_SHELL_ENVS);
+ String[] envs = ActionUtils.getStrings(actionConf, CONF_OOZIE_SHELL_ENVS);
for (String env : envs) {
String[] varValue = env.split("=",2); // Error case is handled in
// ShellActionExecutor
@@ -339,7 +339,7 @@ public class ShellMain extends LauncherMain {
*/
protected List<String> getShellArguments(Configuration actionConf) {
List<String> arguments = new ArrayList<String>();
- String[] scrArgs = MapReduceMain.getStrings(actionConf, CONF_OOZIE_SHELL_ARGS);
+ String[] scrArgs = ActionUtils.getStrings(actionConf, CONF_OOZIE_SHELL_ARGS);
for (String scrArg : scrArgs) {
arguments.add(scrArg);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index c00b263..0029dd0 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -150,7 +150,7 @@ public class PigMain extends LauncherMain {
arguments.add("-file");
arguments.add(script);
- String[] params = MapReduceMain.getStrings(actionConf, PigActionExecutor.PIG_PARAMS);
+ String[] params = ActionUtils.getStrings(actionConf, PigActionExecutor.PIG_PARAMS);
for (String param : params) {
arguments.add("-param");
arguments.add(param);
@@ -204,7 +204,7 @@ public class PigMain extends LauncherMain {
arguments.add("-logfile");
arguments.add(pigLog);
- String[] pigArgs = MapReduceMain.getStrings(actionConf, PigActionExecutor.PIG_ARGS);
+ String[] pigArgs = ActionUtils.getStrings(actionConf, PigActionExecutor.PIG_ARGS);
for (String pigArg : pigArgs) {
if (DISALLOWED_PIG_OPTIONS.contains(pigArg)) {
throw new RuntimeException("Error: Pig argument " + pigArg + " is not supported");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
index b84e2b5..52aabe7 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
@@ -136,7 +136,7 @@ public class PigMainWithOldAPI extends LauncherMain {
arguments.add("-file");
arguments.add(script);
- String[] params = MapReduceMain.getStrings(actionConf, "oozie.pig.params");
+ String[] params = ActionUtils.getStrings(actionConf, "oozie.pig.params");
for (String param : params) {
arguments.add("-param");
arguments.add(param);
@@ -188,7 +188,7 @@ public class PigMainWithOldAPI extends LauncherMain {
arguments.add("-logfile");
arguments.add(pigLog);
- String[] pigArgs = MapReduceMain.getStrings(actionConf, "oozie.pig.args");
+ String[] pigArgs = ActionUtils.getStrings(actionConf, "oozie.pig.args");
for (String pigArg : pigArgs) {
if (DISALLOWED_PIG_OPTIONS.contains(pigArg)) {
throw new RuntimeException("Error: Pig argument " + pigArg + " is not supported");
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
index e52e6fd..74de433 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
@@ -21,7 +21,6 @@ package org.apache.oozie.action.hadoop;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.action.hadoop.MainTestCase;
-import org.apache.oozie.action.hadoop.MapReduceMain;
import org.apache.oozie.action.hadoop.PigMainWithOldAPI;
import org.apache.oozie.action.hadoop.SharelibUtils;
import org.apache.oozie.test.XFsTestCase;
@@ -97,9 +96,9 @@ public class TestPigMainWithOldAPI extends XFsTestCase implements Callable<Void>
SharelibUtils.addToDistributedCache("pig", fs, getFsTestCaseDir(), jobConfiguration);
String[] params = { "IN=" + inputDir.toUri().getPath(), "OUT=" + outputDir.toUri().getPath() };
- MapReduceMain.setStrings(jobConfiguration, "oozie.pig.params", params);
+ ActionUtils.setStrings(jobConfiguration, "oozie.pig.params", params);
String[] args = { "-v" };
- MapReduceMain.setStrings(jobConfiguration, "oozie.pig.args", args);
+ ActionUtils.setStrings(jobConfiguration, "oozie.pig.args", args);
File actionXml = new File(getTestCaseDir(), "action.xml");
OutputStream os = new FileOutputStream(actionXml);
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index 9e53af5..9c6ec6c 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -161,7 +161,7 @@ public class SqoopMain extends LauncherMain {
Configuration sqoopConf = setUpSqoopSite();
String logFile = setUpSqoopLog4J(sqoopConf);
- String[] sqoopArgs = MapReduceMain.getStrings(sqoopConf, SqoopActionExecutor.SQOOP_ARGS);
+ String[] sqoopArgs = ActionUtils.getStrings(sqoopConf, SqoopActionExecutor.SQOOP_ARGS);
if (sqoopArgs == null) {
throw new RuntimeException("Action Configuration does not have [" + SqoopActionExecutor.SQOOP_ARGS + "] property");
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/165847cc/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java
----------------------------------------------------------------------
diff --git a/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java b/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java
index 991bf7e..cc55166 100644
--- a/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java
+++ b/sharelib/streaming/src/main/java/org/apache/oozie/action/hadoop/StreamingMain.java
@@ -56,12 +56,12 @@ public class StreamingMain extends MapReduceMain {
if (value != null) {
jobConf.set("stream.recordreader.class", value);
}
- String[] values = getStrings(actionConf, "oozie.streaming.record-reader-mapping");
+ String[] values = ActionUtils.getStrings(actionConf, "oozie.streaming.record-reader-mapping");
for (String s : values) {
String[] kv = s.split("=");
jobConf.set("stream.recordreader." + kv[0], kv[1]);
}
- values = getStrings(actionConf, "oozie.streaming.env");
+ values = ActionUtils.getStrings(actionConf, "oozie.streaming.env");
value = jobConf.get("stream.addenvironment", "");
if (value.length() > 0) {
value = value + " ";