You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/02/04 19:02:22 UTC
svn commit: r1442242 - in /oozie/branches/branch-3.3: ./
core/src/main/java/org/apache/oozie/workflow/lite/
core/src/test/java/org/apache/oozie/workflow/lite/ core/src/test/resources/
docs/src/site/twiki/
Author: rkanter
Date: Mon Feb 4 18:02:22 2013
New Revision: 1442242
URL: http://svn.apache.org/viewvc?rev=1442242&view=rev
Log:
OOZIE-1034 Allow disabling forkjoin validation just for a specific workflow (rkanter)
Added:
oozie/branches/branch-3.3/core/src/test/resources/wf-invalid-fork.xml
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java?rev=1442242&r1=1442241&r2=1442242&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java Mon Feb 4 18:02:22 2013
@@ -82,6 +82,7 @@ public class LiteWorkflowAppParser {
private static final String KILL_MESSAGE_E = "message";
public static final String VALIDATE_FORK_JOIN = "oozie.validate.ForkJoin";
+ public static final String WF_VALIDATE_FORK_JOIN = "oozie.wf.validate.ForkJoin";
private Schema schema;
private Class<? extends ControlNodeHandler> controlNodeHandler;
@@ -130,7 +131,7 @@ public class LiteWorkflowAppParser {
traversed.put(app.getNode(StartNodeDef.START).getName(), VisitStatus.VISITING);
validate(app, app.getNode(StartNodeDef.START), traversed);
//Validate whether fork/join are in pair or not
- if (Services.get().getConf().getBoolean(VALIDATE_FORK_JOIN, true)) {
+ if (jobConf.getBoolean(WF_VALIDATE_FORK_JOIN, true) && Services.get().getConf().getBoolean(VALIDATE_FORK_JOIN, true)) {
validateForkJoin(app);
}
return app;
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java?rev=1442242&r1=1442241&r2=1442242&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java Mon Feb 4 18:02:22 2013
@@ -941,4 +941,75 @@ public class TestLiteWorkflowAppParser e
done = true;
}
}
+
+ public void testDisableWFValidateForkJoin() throws Exception {
+ LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
+ LiteWorkflowStoreService.LiteControlNodeHandler.class,
+ LiteWorkflowStoreService.LiteDecisionHandler.class,
+ LiteWorkflowStoreService.LiteActionHandler.class);
+
+ // oozie level default, wf level default
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+
+ // oozie level default, wf level disabled
+ Configuration conf = new Configuration();
+ conf.set("oozie.wf.validate.ForkJoin", "false");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level default, wf level enabled
+ conf.set("oozie.wf.validate.ForkJoin", "true");
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+
+ // oozie level disabled, wf level default
+ Services.get().destroy();
+ setSystemProperty("oozie.validate.ForkJoin", "false");
+ new Services().init();
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+
+ // oozie level disabled, wf level disabled
+ conf.set("oozie.wf.validate.ForkJoin", "false");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level disabled, wf level enabled
+ conf.set("oozie.wf.validate.ForkJoin", "true");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level enabled, wf level default
+ Services.get().destroy();
+ setSystemProperty("oozie.validate.ForkJoin", "true");
+ new Services().init();
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+
+ // oozie level enabled, wf level disabled
+ conf.set("oozie.wf.validate.ForkJoin", "false");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level enabled, wf level enabled
+ conf.set("oozie.wf.validate.ForkJoin", "true");
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+ }
}
Added: oozie/branches/branch-3.3/core/src/test/resources/wf-invalid-fork.xml
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/resources/wf-invalid-fork.xml?rev=1442242&view=auto
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/resources/wf-invalid-fork.xml (added)
+++ oozie/branches/branch-3.3/core/src/test/resources/wf-invalid-fork.xml Mon Feb 4 18:02:22 2013
@@ -0,0 +1,64 @@
+<!--
+ Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.1" name="wf-name">
+ <start to="a"/>
+
+ <fork name="a">
+ <path start="b1"/>
+ <path start="b2"/>
+ </fork>
+
+ <action name="b1">
+ <sub-workflow>
+ <app-path>foo/bar</app-path>
+ </sub-workflow>
+ <ok to="c"/>
+ <error to="fail"/>
+ </action>
+
+ <action name="b2">
+ <map-reduce>
+ <job-tracker>foo</job-tracker>
+ <name-node>bar</name-node>
+ <streaming>
+ <mapper>foo.sh</mapper>
+ <reducer>bar.sh</reducer>
+ </streaming>
+ </map-reduce>
+
+ <ok to="c"/>
+ <error to="fail"/>
+ </action>
+
+ <action name="c">
+ <map-reduce>
+ <job-tracker>foo</job-tracker>
+ <name-node>bar</name-node>
+ <streaming>
+ <mapper>foo.sh</mapper>
+ <reducer>bar.sh</reducer>
+ </streaming>
+ </map-reduce>
+
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>fail</message>
+ </kill>
+
+ <end name="end"/>
+</workflow-app>
Modified: oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1442242&r1=1442241&r2=1442242&view=diff
==============================================================================
--- oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki (original)
+++ oozie/branches/branch-3.3/docs/src/site/twiki/WorkflowFunctionalSpec.twiki Mon Feb 4 18:02:22 2013
@@ -457,6 +457,14 @@ fork arrive to the join node.
</workflow-app>
</verbatim>
+By default, Oozie performs some validation that any forking in a workflow is valid and won't lead to any incorrect behavior or
+instability. However, if Oozie is preventing a workflow from being submitted and you are very certain that it should work, you can
+disable forkjoin validation so that Oozie will accept the workflow. To disable this validation just for a specific workflow, simply
+set =oozie.wf.validate.ForkJoin= to =false= in the job.properties file. To disable this validation for all workflows, simply set
+=oozie.validate.ForkJoin= to =false= in the oozie-site.xml file. Disabling this validation is determined by the AND of both of
+these properties, so it will be disabled if either or both are set to false and only enabled if both are set to true (or not
+specified).
+
#ActionNodes
---+++ 3.2 Workflow Action Nodes
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1442242&r1=1442241&r2=1442242&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Mon Feb 4 18:02:22 2013
@@ -1,5 +1,6 @@
-- Oozie 3.3.2 (unreleased)
+OOZIE-1034 Allow disabling forkjoin validation just for a specific workflow
OOZIE-1072 Oozie Coordinator Doc error in Synchronous datasets
OOZIE-1028 Add EL function to allow date ranges to be used for dataset ranges (rkanter via tucu)
OOZIE-1048 Enable propagation of native libraries as a VM argument using java.library.path (venkatesh via tucu)