You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by mo...@apache.org on 2012/11/29 02:25:34 UTC
svn commit: r1415019 - in /oozie/branches/hcat-intre: ./
core/src/main/java/org/apache/oozie/workflow/lite/
core/src/test/java/org/apache/oozie/workflow/lite/ core/src/test/resources/
docs/src/site/twiki/ examples/src/main/apps/shell/
Author: mona
Date: Thu Nov 29 01:25:33 2012
New Revision: 1415019
URL: http://svn.apache.org/viewvc?rev=1415019&view=rev
Log:
applying trunk patches to hcat-intre branch for sync
Added:
oozie/branches/hcat-intre/core/src/test/resources/wf-invalid-fork.xml
oozie/branches/hcat-intre/examples/src/main/apps/shell/
oozie/branches/hcat-intre/examples/src/main/apps/shell/job.properties
oozie/branches/hcat-intre/examples/src/main/apps/shell/workflow.xml
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
oozie/branches/hcat-intre/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
oozie/branches/hcat-intre/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
oozie/branches/hcat-intre/release-log.txt
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java?rev=1415019&r1=1415018&r2=1415019&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java Thu Nov 29 01:25:33 2012
@@ -82,6 +82,7 @@ public class LiteWorkflowAppParser {
private static final String KILL_MESSAGE_E = "message";
public static final String VALIDATE_FORK_JOIN = "oozie.validate.ForkJoin";
+ public static final String WF_VALIDATE_FORK_JOIN = "oozie.wf.validate.ForkJoin";
private Schema schema;
private Class<? extends ControlNodeHandler> controlNodeHandler;
@@ -130,7 +131,7 @@ public class LiteWorkflowAppParser {
traversed.put(app.getNode(StartNodeDef.START).getName(), VisitStatus.VISITING);
validate(app, app.getNode(StartNodeDef.START), traversed);
//Validate whether fork/join are in pair or not
- if (Services.get().getConf().getBoolean(VALIDATE_FORK_JOIN, true)) {
+ if (jobConf.getBoolean(WF_VALIDATE_FORK_JOIN, true) && Services.get().getConf().getBoolean(VALIDATE_FORK_JOIN, true)) {
validateForkJoin(app);
}
return app;
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java?rev=1415019&r1=1415018&r2=1415019&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java Thu Nov 29 01:25:33 2012
@@ -941,4 +941,75 @@ public class TestLiteWorkflowAppParser e
done = true;
}
}
+
+ public void testDisableWFValidateForkJoin() throws Exception {
+ LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null,
+ LiteWorkflowStoreService.LiteControlNodeHandler.class,
+ LiteWorkflowStoreService.LiteDecisionHandler.class,
+ LiteWorkflowStoreService.LiteActionHandler.class);
+
+ // oozie level default, wf level default
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+
+ // oozie level default, wf level disabled
+ Configuration conf = new Configuration();
+ conf.set("oozie.wf.validate.ForkJoin", "false");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level default, wf level enabled
+ conf.set("oozie.wf.validate.ForkJoin", "true");
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+
+ // oozie level disabled, wf level default
+ Services.get().destroy();
+ setSystemProperty("oozie.validate.ForkJoin", "false");
+ new Services().init();
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+
+ // oozie level disabled, wf level disabled
+ conf.set("oozie.wf.validate.ForkJoin", "false");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level disabled, wf level enabled
+ conf.set("oozie.wf.validate.ForkJoin", "true");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level enabled, wf level default
+ Services.get().destroy();
+ setSystemProperty("oozie.validate.ForkJoin", "true");
+ new Services().init();
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+
+ // oozie level enabled, wf level disabled
+ conf.set("oozie.wf.validate.ForkJoin", "false");
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), conf);
+
+ // oozie level enabled, wf level enabled
+ conf.set("oozie.wf.validate.ForkJoin", "true");
+ try {
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-invalid-fork.xml", -1), new Configuration());
+ }
+ catch (WorkflowException wfe) {
+ assertEquals(ErrorCode.E0730, wfe.getErrorCode());
+ assertEquals("E0730: Fork/Join not in pair", wfe.getMessage());
+ }
+ }
}
Added: oozie/branches/hcat-intre/core/src/test/resources/wf-invalid-fork.xml
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/resources/wf-invalid-fork.xml?rev=1415019&view=auto
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/resources/wf-invalid-fork.xml (added)
+++ oozie/branches/hcat-intre/core/src/test/resources/wf-invalid-fork.xml Thu Nov 29 01:25:33 2012
@@ -0,0 +1,64 @@
+<!--
+ Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.1" name="wf-name">
+ <start to="a"/>
+
+ <fork name="a">
+ <path start="b1"/>
+ <path start="b2"/>
+ </fork>
+
+ <action name="b1">
+ <sub-workflow>
+ <app-path>foo/bar</app-path>
+ </sub-workflow>
+ <ok to="c"/>
+ <error to="fail"/>
+ </action>
+
+ <action name="b2">
+ <map-reduce>
+ <job-tracker>foo</job-tracker>
+ <name-node>bar</name-node>
+ <streaming>
+ <mapper>foo.sh</mapper>
+ <reducer>bar.sh</reducer>
+ </streaming>
+ </map-reduce>
+
+ <ok to="c"/>
+ <error to="fail"/>
+ </action>
+
+ <action name="c">
+ <map-reduce>
+ <job-tracker>foo</job-tracker>
+ <name-node>bar</name-node>
+ <streaming>
+ <mapper>foo.sh</mapper>
+ <reducer>bar.sh</reducer>
+ </streaming>
+ </map-reduce>
+
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>fail</message>
+ </kill>
+
+ <end name="end"/>
+</workflow-app>
Modified: oozie/branches/hcat-intre/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki?rev=1415019&r1=1415018&r2=1415019&view=diff
==============================================================================
--- oozie/branches/hcat-intre/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki (original)
+++ oozie/branches/hcat-intre/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki Thu Nov 29 01:25:33 2012
@@ -432,7 +432,7 @@ The following EL constants can be used w
| =MONTH= | _DD_ | 2 digits representing the month of the year, January = 1 |
| =DAY= | _DD_ | 2 digits representing the day of the month |
| =HOUR= | _HH_ | 2 digits representing the hour of the day, in 24 hour format, 0 - 23 |
-| =MIN= | _mm_ | 2 digits reprensenting the minute of the hour, 0 - 59 |
+| =MINUTE= | _mm_ | 2 digits reprensenting the minute of the hour, 0 - 59 |
*%PURPLE% Syntax: %ENDCOLOR%*
Modified: oozie/branches/hcat-intre/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1415019&r1=1415018&r2=1415019&view=diff
==============================================================================
--- oozie/branches/hcat-intre/docs/src/site/twiki/WorkflowFunctionalSpec.twiki (original)
+++ oozie/branches/hcat-intre/docs/src/site/twiki/WorkflowFunctionalSpec.twiki Thu Nov 29 01:25:33 2012
@@ -457,6 +457,14 @@ fork arrive to the join node.
</workflow-app>
</verbatim>
+By default, Oozie performs some validation that any forking in a workflow is valid and won't lead to any incorrect behavior or
+instability. However, if Oozie is preventing a workflow from being submitted and you are very certain that it should work, you can
+disable forkjoin validation so that Oozie will accept the workflow. To disable this validation just for a specific workflow, simply
+set =oozie.wf.validate.ForkJoin= to =false= in the job.properties file. To disable this validation for all workflows, simply set
+=oozie.validate.ForkJoin= to =false= in the oozie-site.xml file. Disabling this validation is determined by the AND of both of
+these properties, so it will be disabled if either or both are set to false and only enabled if both are set to true (or not
+specified).
+
#ActionNodes
---+++ 3.2 Workflow Action Nodes
Added: oozie/branches/hcat-intre/examples/src/main/apps/shell/job.properties
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/examples/src/main/apps/shell/job.properties?rev=1415019&view=auto
==============================================================================
--- oozie/branches/hcat-intre/examples/src/main/apps/shell/job.properties (added)
+++ oozie/branches/hcat-intre/examples/src/main/apps/shell/job.properties Thu Nov 29 01:25:33 2012
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+examplesRoot=examples
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/shell
Added: oozie/branches/hcat-intre/examples/src/main/apps/shell/workflow.xml
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/examples/src/main/apps/shell/workflow.xml?rev=1415019&view=auto
==============================================================================
--- oozie/branches/hcat-intre/examples/src/main/apps/shell/workflow.xml (added)
+++ oozie/branches/hcat-intre/examples/src/main/apps/shell/workflow.xml Thu Nov 29 01:25:33 2012
@@ -0,0 +1,52 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.4" name="shell-wf">
+ <start to="shell-node"/>
+ <action name="shell-node">
+ <shell xmlns="uri:oozie:shell-action:0.2">
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ </configuration>
+ <exec>echo</exec>
+ <argument>my_output=Hello Oozie</argument>
+ <capture-output/>
+ </shell>
+ <ok to="check-output"/>
+ <error to="fail"/>
+ </action>
+ <decision name="check-output">
+ <switch>
+ <case to="end">
+ ${wf:actionData('shell-node')['my_output'] eq 'Hello Oozie'}
+ </case>
+ <default to="fail-output"/>
+ </switch>
+ </decision>
+ <kill name="fail">
+ <message>Shell action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <kill name="fail-output">
+ <message>Incorrect output, expected [Hello Oozie] but was [${wf:actionData('shell-node')['my_output']}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
Modified: oozie/branches/hcat-intre/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1415019&r1=1415018&r2=1415019&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Thu Nov 29 01:25:33 2012
@@ -9,6 +9,9 @@ OOZIE-1061 Add new EL functions to retri
OOZIE-1056 Command to update push-based dependency (mohammad)
OOZIE-1059 Add static method to create URI String in HCatURI(ryota via mohammad)
OOZIE-1039 Implement the Missing Dependency structure for HCat partitions (mona via mohammad)
+OOZIE-1062 Create a shell example
+OOZIE-1034 Allow disabling forkjoin validation just for a specific workflow
+OOZIE-1072 Oozie Coordinator Doc error in Synchronous datasets
OOZIE-1028 Add EL function to allow date ranges to be used for dataset ranges (rkanter via tucu)
OOZIE-1045 Parameterize <unresolved-instances> tag currently hardcoded (egashira via mona)
OOZIE-1029 MiniMRCluster fails to start when used against YARN (rkanter via tucu)