You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2012/08/08 09:29:16 UTC
svn commit: r1370682 - in /incubator/oozie/trunk: ./
core/src/main/java/org/apache/oozie/action/
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/main/java/org/apache/oozie/workflow/lite/
core/src/test/java/org/apache/oozie/workflow/lite/
Author: virag
Date: Wed Aug 8 07:29:15 2012
New Revision: 1370682
URL: http://svn.apache.org/viewvc?rev=1370682&view=rev
Log:
OOZIE-921 Changes in global section for the Name Node in FS action (bcyr via virag)
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/ActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/ActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/ActionExecutor.java?rev=1370682&r1=1370681&r2=1370682&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/ActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/ActionExecutor.java Wed Aug 8 07:29:15 2012
@@ -52,6 +52,8 @@ public abstract class ActionExecutor {
* Error code used by {@link #convertException} when there is not register error information for an exception.
*/
public static final String ERROR_OTHER = "OTHER";
+
+ public boolean requiresNNJT = false;
private static class ErrorInfo {
ActionExecutorException.ErrorType errorType;
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1370682&r1=1370681&r2=1370682&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java Wed Aug 8 07:29:15 2012
@@ -105,10 +105,12 @@ public class JavaActionExecutor extends
public JavaActionExecutor() {
this("java");
+ requiresNNJT = true;
}
protected JavaActionExecutor(String type) {
super(type);
+ requiresNNJT = true;
}
protected String getLauncherJarName() {
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java?rev=1370682&r1=1370681&r2=1370682&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java Wed Aug 8 07:29:15 2012
@@ -24,6 +24,7 @@ import org.apache.oozie.util.ParamChecke
import org.apache.oozie.util.ParameterVerifier;
import org.apache.oozie.util.ParameterVerifierException;
import org.apache.oozie.ErrorCode;
+import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.ActionService;
import org.apache.hadoop.conf.Configuration;
@@ -448,7 +449,7 @@ public class LiteWorkflowAppParser {
// Use the action's namespace when getting children of the action (will be different than ns for extension actions)
Namespace actionNs = eActionConf.getNamespace();
-
+
if (global != null) {
Element globalJobTracker = global.getChild("job-tracker", ns);
Element globalNameNode = global.getChild("name-node", ns);
@@ -457,26 +458,20 @@ public class LiteWorkflowAppParser {
if (globalJobTracker != null && eActionConf.getChild("job-tracker", actionNs) == null) {
Element jobTracker = new Element("job-tracker", actionNs);
jobTracker.setText(globalJobTracker.getText());
- eActionConf.addContent(0, jobTracker);
+ eActionConf.addContent(jobTracker);
}
if (globalNameNode != null && eActionConf.getChild("name-node", actionNs) == null) {
Element nameNode = new Element("name-node", actionNs);
nameNode.setText(globalNameNode.getText());
- eActionConf.addContent(1, nameNode);
+ eActionConf.addContent(nameNode);
}
if (globalConfiguration != null) {
Element actionConfiguration = eActionConf.getChild("configuration", actionNs);
if (actionConfiguration == null) {
actionConfiguration = new Element("configuration", actionNs);
- int index = 2;
- index += eActionConf.getChild("prepare", actionNs) == null ? 0 : 2;
- index += eActionConf.getChild("job-xml", actionNs) == null ? 0 : 2;
- index += eActionConf.getChild("pipes", actionNs) == null ? 0 : 2;
- index += eActionConf.getChild("streaming", actionNs) == null ? 0 : 2;
-
- eActionConf.addContent(index, actionConfiguration);
+ eActionConf.addContent(actionConfiguration);
}
for (Element globalConfig : (List<Element>) globalConfiguration.getChildren()) {
boolean isSet = false;
@@ -503,10 +498,11 @@ public class LiteWorkflowAppParser {
}
}
else {
- if (eActionConf.getName().equalsIgnoreCase("pig") || eActionConf.getName().equalsIgnoreCase("map-reduce")
- || eActionConf.getName().equalsIgnoreCase("java")
- || eActionConf.getName().equalsIgnoreCase("shell")
- || eActionConf.getName().equalsIgnoreCase("hive")) {
+ ActionExecutor ae = Services.get().get(ActionService.class).getExecutor(eActionConf.getName());
+ if (ae == null) {
+ throw new WorkflowException(ErrorCode.E0723, "Unsupported action type");
+ }
+ if (ae.requiresNNJT) {
if (eActionConf.getChild("name-node", actionNs) == null) {
throw new WorkflowException(ErrorCode.E0701, "No name-node defined");
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java?rev=1370682&r1=1370681&r2=1370682&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java Wed Aug 8 07:29:15 2012
@@ -66,14 +66,12 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1),
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1),
new Configuration());
String d = app.getNode("d").getConf();
String expectedD =
"<map-reduce xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
- " <job-tracker>foo</job-tracker>\r\n" +
- " <name-node>bar</name-node>\r\n" +
" <prepare>\r\n" +
" <delete path=\"/tmp\" />\r\n" +
" <mkdir path=\"/tmp\" />\r\n" +
@@ -83,6 +81,10 @@ public class TestLiteWorkflowAppParser e
" <reducer>/mywc.sh</reducer>\r\n" +
" </streaming>\r\n" +
" <job-xml>/tmp</job-xml>\r\n" +
+ " <file>/tmp</file>\r\n" +
+ " <archive>/tmp</archive>\r\n" +
+ " <job-tracker>foo</job-tracker>\r\n" +
+ " <name-node>bar</name-node>\r\n" +
" <configuration>\r\n" +
" <property>\r\n" +
" <name>a</name>\r\n" +
@@ -93,8 +95,6 @@ public class TestLiteWorkflowAppParser e
" <value>B</value>\r\n" +
" </property>\r\n" +
" </configuration>\r\n" +
- " <file>/tmp</file>\r\n" +
- " <archive>/tmp</archive>\r\n" +
"</map-reduce>";
assertEquals(expectedD.replaceAll(" ",""), d.replaceAll(" ", ""));
@@ -106,14 +106,12 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1),
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1),
new Configuration());
String e = app.getNode("e").getConf();
String expectedE =
"<pig xmlns=\"uri:oozie:workflow:0.4\">\r\n" +
- " <job-tracker>foo</job-tracker>\r\n" +
- " <name-node>bar</name-node>\r\n" +
" <prepare>\r\n" +
" <delete path=\"/tmp\" />\r\n" +
" <mkdir path=\"/tmp\" />\r\n" +
@@ -132,25 +130,25 @@ public class TestLiteWorkflowAppParser e
" <param>x</param>\r\n" +
" <file>/tmp</file>\r\n" +
" <file>/tmp</file>\r\n" +
+ " <job-tracker>foo</job-tracker>\r\n" +
+ " <name-node>bar</name-node>\r\n" +
"</pig>";
assertEquals(expectedE.replaceAll(" ", ""), e.replaceAll(" ", ""));
}
-
+
public void testParserGlobalExtensionActions() throws Exception {
LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
LiteWorkflowStoreService.LiteControlNodeHandler.class,
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1),
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1),
new Configuration());
String a = app.getNode("a").getConf();
String expectedA =
"<hive xmlns=\"uri:oozie:hive-action:0.2\">\r\n" +
- " <job-tracker>foo</job-tracker>\r\n" +
- " <name-node>bar</name-node>\r\n" +
" <prepare>\r\n" +
" <delete path=\"/tmp\" />\r\n" +
" <mkdir path=\"/tmp\" />\r\n" +
@@ -172,18 +170,20 @@ public class TestLiteWorkflowAppParser e
" <script>script.q</script>\r\n" +
" <param>INPUT=/tmp/table</param>\r\n" +
" <param>OUTPUT=/tmp/hive</param>\r\n" +
+ " <job-tracker>foo</job-tracker>\r\n" +
+ " <name-node>bar</name-node>\r\n" +
"</hive>";
System.out.println("AAA " + expectedA.replaceAll(" ", ""));
assertEquals(expectedA.replaceAll(" ",""), a.replaceAll(" ", ""));
}
-
+
public void testParserGlobalExtensionActionsLocalAlreadyExists() throws Exception {
LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
LiteWorkflowStoreService.LiteControlNodeHandler.class,
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1),
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1),
new Configuration());
String b = app.getNode("b").getConf();
@@ -210,20 +210,20 @@ public class TestLiteWorkflowAppParser e
"</distcp>";
assertEquals(expectedB.replaceAll(" ",""), b.replaceAll(" ", ""));
}
-
+
public void testParserGlobalExtensionActionsNoGlobal() throws Exception {
LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
LiteWorkflowStoreService.LiteControlNodeHandler.class,
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
-
+
// If no global section is defined, some extension actions (e.g. hive) must still have name-node and job-tracker elements
// or the handleGlobal() method will throw an exception
-
+
parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext-no-global.xml", -1), new Configuration());
-
+
try {
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-invalid-global-ext-no-global.xml", -1),
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-invalid-global-ext-no-global.xml", -1),
new Configuration());
fail();
}
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1370682&r1=1370681&r2=1370682&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Wed Aug 8 07:29:15 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-921 Changes in global section for the Name Node in FS action (bcyr via virag)
OOZIE-942 Add formal Parameters to bundle XML (rkanter via virag)
OOZIE-239 Add formal parameters to WF & COORD XML (rkanter via tucu)
OOZIE-938 Remove some duplicated code in FsActionExecutor (rkanter via tucu)