You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/08/07 06:39:51 UTC
svn commit: r1370117 - in /incubator/oozie/trunk: ./
client/src/main/java/org/apache/oozie/cli/ client/src/main/resources/
core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/command/coord/
core/src/main/java/org/apache/oozie/comma...
Author: tucu
Date: Tue Aug 7 04:39:50 2012
New Revision: 1370117
URL: http://svn.apache.org/viewvc?rev=1370117&view=rev
Log:
OOZIE-239 Add formal parameters to WF & COORD XML (rkanter via tucu)
Added:
incubator/oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifier.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifierException.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParameterVerifier.java
Modified:
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
incubator/oozie/trunk/client/src/main/resources/oozie-workflow-0.4.xsd
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/LiteWorkflowAppService.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/WorkflowLib.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowLib.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java (original)
+++ incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Tue Aug 7 04:39:50 2012
@@ -1446,6 +1446,8 @@ public class OozieCLI {
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"oozie-coordinator-0.3.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
+ "oozie-coordinator-0.4.xsd")));
+ sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"oozie-bundle-0.1.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"oozie-sla-0.1.xsd")));
Added: incubator/oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd?rev=1370117&view=auto
==============================================================================
--- incubator/oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd (added)
+++ incubator/oozie/trunk/client/src/main/resources/oozie-coordinator-0.4.xsd Tue Aug 7 04:39:50 2012
@@ -0,0 +1,147 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:coordinator="uri:oozie:coordinator:0.4"
+ elementFormDefault="qualified" targetNamespace="uri:oozie:coordinator:0.4">
+
+ <xs:element name="coordinator-app" type="coordinator:COORDINATOR-APP"/>
+ <xs:element name="datasets" type="coordinator:DATASETS"/>
+ <xs:simpleType name="IDENTIFIER">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="([a-zA-Z]([\-_a-zA-Z0-9])*){1,39}"/>
+ </xs:restriction>
+ </xs:simpleType>
+ <xs:complexType name="COORDINATOR-APP">
+ <xs:sequence>
+ <xs:element name="parameters" type="coordinator:PARAMETERS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="controls" type="coordinator:CONTROLS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="datasets" type="coordinator:DATASETS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="input-events" type="coordinator:INPUTEVENTS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="output-events" type="coordinator:OUTPUTEVENTS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="action" type="coordinator:ACTION" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="frequency" type="xs:string" use="required"/>
+ <xs:attribute name="start" type="xs:string" use="required"/>
+ <xs:attribute name="end" type="xs:string" use="required"/>
+ <xs:attribute name="timezone" type="xs:string" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="PARAMETERS">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="CONTROLS">
+ <xs:sequence minOccurs="0" maxOccurs="1">
+ <xs:element name="timeout" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="concurrency" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="execution" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="throttle" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="DATASETS">
+ <xs:sequence minOccurs="0" maxOccurs="1">
+ <xs:element name="include" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="dataset" type="coordinator:SYNCDATASET" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="async-dataset" type="coordinator:ASYNCDATASET" minOccurs="0" maxOccurs="1"/>
+ </xs:choice>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="SYNCDATASET">
+ <xs:sequence>
+ <xs:element name="uri-template" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="done-flag" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="frequency" type="xs:string" use="required"/>
+ <xs:attribute name="initial-instance" type="xs:string" use="required"/>
+ <xs:attribute name="timezone" type="xs:string" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="ASYNCDATASET">
+ <xs:sequence>
+ <xs:element name="uri-template" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="sequence-type" type="xs:string" use="required"/>
+ <xs:attribute name="initial-version" type="xs:string" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="INPUTEVENTS">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="data-in" type="coordinator:DATAIN" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="DATAIN">
+ <xs:choice minOccurs="1" maxOccurs="1">
+ <xs:element name="instance" type="xs:string" minOccurs="1" maxOccurs="unbounded"/>
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="start-instance" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="end-instance" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:choice>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="dataset" type="coordinator:IDENTIFIER" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="OUTPUTEVENTS">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="data-out" type="coordinator:DATAOUT" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="DATAOUT">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="instance" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="dataset" type="coordinator:IDENTIFIER" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="ACTION">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="workflow" type="coordinator:WORKFLOW" minOccurs="1" maxOccurs="1"/>
+ <xs:any namespace="uri:oozie:sla:0.1" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="WORKFLOW">
+ <xs:sequence>
+ <xs:element name="app-path" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="configuration" type="coordinator:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="FLAG"/>
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+</xs:schema>
Modified: incubator/oozie/trunk/client/src/main/resources/oozie-workflow-0.4.xsd
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/resources/oozie-workflow-0.4.xsd?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/client/src/main/resources/oozie-workflow-0.4.xsd (original)
+++ incubator/oozie/trunk/client/src/main/resources/oozie-workflow-0.4.xsd Tue Aug 7 04:39:50 2012
@@ -29,8 +29,9 @@
<xs:complexType name="WORKFLOW-APP">
<xs:sequence>
- <xs:element name="global" type="workflow:GLOBAL" minOccurs="0" maxOccurs="1"/>
- <xs:element name="credentials" type="workflow:CREDENTIALS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="parameters" type="workflow:PARAMETERS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="global" type="workflow:GLOBAL" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="credentials" type="workflow:CREDENTIALS" minOccurs="0" maxOccurs="1"/>
<xs:element name="start" type="workflow:START" minOccurs="1" maxOccurs="1"/>
<xs:choice minOccurs="0" maxOccurs="unbounded">
<xs:element name="decision" type="workflow:DECISION" minOccurs="1" maxOccurs="1"/>
@@ -45,6 +46,20 @@
<xs:attribute name="name" type="xs:string" use="required"/>
</xs:complexType>
+ <xs:complexType name="PARAMETERS">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
<xs:complexType name="GLOBAL">
<xs:sequence>
<xs:element name="job-tracker" type="xs:string" minOccurs="1" maxOccurs="1"/>
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java Tue Aug 7 04:39:50 2012
@@ -143,6 +143,7 @@ public enum ErrorCode {
E0735(XLog.STD, "There was an invalid \"error to\" transition to node [{1}] while using fork/join"),
E0736(XLog.STD, "Workflow definition lenght [{0}] exceeded maximum allowed length [{1}]"),
E0737(XLog.STD, "Invalid transition from node [{0}] to node [{1}] while using Fork/Join because node [{1}] is of type [{2}]"),
+ E0738(XLog.STD, "The following {0} parameters are required but were not defined and no default values are available: {1}"),
E0800(XLog.STD, "Action it is not running its in [{1}] state, action [{0}]"),
E0801(XLog.STD, "Workflow already running, workflow [{0}]"),
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java Tue Aug 7 04:39:50 2012
@@ -77,6 +77,8 @@ import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.InstrumentUtils;
import org.apache.oozie.util.LogUtils;
import org.apache.oozie.util.ParamChecker;
+import org.apache.oozie.util.ParameterVerifier;
+import org.apache.oozie.util.ParameterVerifierException;
import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
@@ -218,6 +220,8 @@ public class CoordSubmitXCommand extends
String appNamespace = readAppNamespace(appXml);
coordJob.setAppNamespace(appNamespace);
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(appXml));
+
appXml = XmlUtils.removeComments(appXml);
initEvaluators();
Element eJob = basicResolveAndIncludeDS(appXml, conf, coordJob);
@@ -274,6 +278,11 @@ public class CoordSubmitXCommand extends
LOG.warn("ERROR: ", cex);
throw new CommandException(cex);
}
+ catch (ParameterVerifierException pex) {
+ exceptionOccured = true;
+ LOG.warn("ERROR: ", pex);
+ throw new CommandException(pex);
+ }
catch (IllegalArgumentException iex) {
exceptionOccured = true;
LOG.warn("ERROR: ", iex);
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/wf/SubmitHttpXCommand.java Tue Aug 7 04:39:50 2012
@@ -108,7 +108,7 @@ public abstract class SubmitHttpXCommand
String wfXml = getWorkflowXml(conf);
LOG.debug("workflow xml created on the server side is :\n");
LOG.debug(wfXml);
- WorkflowApp app = wps.parseDef(wfXml);
+ WorkflowApp app = wps.parseDef(wfXml, conf);
XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken, false);
WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/LiteWorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/LiteWorkflowAppService.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/LiteWorkflowAppService.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/LiteWorkflowAppService.java Tue Aug 7 04:39:50 2012
@@ -39,11 +39,11 @@ public class LiteWorkflowAppService exte
String appPath = ParamChecker.notEmpty(jobConf.get(OozieClient.APP_PATH), OozieClient.APP_PATH);
String user = ParamChecker.notEmpty(jobConf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
String workflowXml = readDefinition(appPath, user, authToken, jobConf);
- return parseDef(workflowXml);
+ return parseDef(workflowXml, jobConf);
}
- public WorkflowApp parseDef(String workflowXml) throws WorkflowException {
+ public WorkflowApp parseDef(String workflowXml, Configuration jobConf) throws WorkflowException {
WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
- return workflowLib.parseDef(workflowXml);
+ return workflowLib.parseDef(workflowXml, jobConf);
}
}
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/SchemaService.java Tue Aug 7 04:39:50 2012
@@ -65,7 +65,8 @@ public class SchemaService implements Se
"oozie-workflow-0.2.5.xsd",
"oozie-workflow-0.3.xsd",
"oozie-workflow-0.4.xsd"};
- private static final String OOZIE_COORDINATOR_XSD[] = { "oozie-coordinator-0.1.xsd", "oozie-coordinator-0.2.xsd", "oozie-coordinator-0.3.xsd"};
+ private static final String OOZIE_COORDINATOR_XSD[] = { "oozie-coordinator-0.1.xsd", "oozie-coordinator-0.2.xsd",
+ "oozie-coordinator-0.3.xsd", "oozie-coordinator-0.4.xsd"};
private static final String OOZIE_BUNDLE_XSD[] = { "oozie-bundle-0.1.xsd" };
private static final String OOZIE_SLA_SEMANTIC_XSD[] = { "gms-oozie-sla-0.1.xsd" };
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java Tue Aug 7 04:39:50 2012
@@ -239,10 +239,11 @@ public abstract class WorkflowAppService
/**
* Parse workflow definition.
* @param wfXml workflow.
+ * @param jobConf job configuration
* @return workflow application.
* @throws WorkflowException thrown if the workflow application could not be parsed.
*/
- public abstract WorkflowApp parseDef(String wfXml) throws WorkflowException;
+ public abstract WorkflowApp parseDef(String wfXml, Configuration jobConf) throws WorkflowException;
/**
* Get all library paths.
Added: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifier.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifier.java?rev=1370117&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifier.java (added)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifier.java Tue Aug 7 04:39:50 2012
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.util;
+
+import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.oozie.ErrorCode;
+import org.jdom.Element;
+import org.jdom.Namespace;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Utility class to parse and verify the <parameters> section in a workflow or coordinator job
+ */
+public abstract class ParameterVerifier
+{
+ private static final Pattern nsVersionPattern = Pattern.compile("uri:oozie:workflow:(\\d+.\\d+)");
+
+ /**
+ * Verify the parameters section
+ *
+ * @param conf The job configuration
+ * @param rootElement The root of the workflow definition
+ * @throws ParameterVerifierException If required parameters are not defined and have no default values, an exception is thrown
+ */
+ public static void verifyParameters(Configuration conf, Element rootElement) throws ParameterVerifierException {
+ ParamChecker.notNull(conf, "conf");
+ if (rootElement == null) {
+ return;
+ }
+
+ Element params = rootElement.getChild("parameters", rootElement.getNamespace());
+ if (params != null) {
+ int numMissing = 0;
+ StringBuilder missingParameters = new StringBuilder();
+ Namespace paramsNs = params.getNamespace();
+ Iterator<Element> it = params.getChildren("property", paramsNs).iterator();
+ while (it.hasNext()) {
+ Element prop = it.next();
+ String name = prop.getChildTextTrim("name", paramsNs);
+ if (name != null) {
+ if (conf.get(name) == null) {
+ String defaultValue = prop.getChildTextTrim("value", paramsNs);
+ if (defaultValue != null) {
+ conf.set(name, defaultValue);
+ } else {
+ missingParameters.append(name);
+ missingParameters.append(", ");
+ numMissing++;
+ }
+ }
+ }
+ }
+ if (numMissing > 0) {
+ missingParameters.setLength(missingParameters.length() - 2); //remove the trailing ", "
+ throw new ParameterVerifierException(ErrorCode.E0738, numMissing, missingParameters.toString());
+ }
+ } else {
+ // If the version is 0.4 or higher, log a warning when the <parameters> section is missing
+ String ns = rootElement.getNamespaceURI();
+ Matcher m = nsVersionPattern.matcher(ns);
+ if (m.matches() && m.groupCount() == 1) {
+ double v = Double.parseDouble(m.group(1));
+ if (v >= 0.4) {
+ XLog.getLog(ParameterVerifier.class).warn("The application does not define formal parameters in its XML "
+ + "definition");
+ }
+ }
+ }
+ }
+}
Added: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifierException.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifierException.java?rev=1370117&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifierException.java (added)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ParameterVerifierException.java Tue Aug 7 04:39:50 2012
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2012 Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.util;
+
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.XException;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Exception thrown by {@link org.apache.oozie.util.ParameterHandler} .
+ */
+public class ParameterVerifierException extends XException {
+
+ /**
+ * Create a ParameterHandler exception from a XException.
+ *
+ * @param cause the XException cause.
+ */
+ public ParameterVerifierException(XException cause) {
+ super(cause);
+ }
+
+ /**
+ * Create a ParameterHandler exception.
+ *
+ * @param errorCode error code.
+ * @param params parameters for the error code message template.
+ */
+ public ParameterVerifierException(ErrorCode errorCode, Object... params) {
+ super(errorCode, params);
+ }
+
+}
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/WorkflowLib.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/WorkflowLib.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/WorkflowLib.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/WorkflowLib.java Tue Aug 7 04:39:50 2012
@@ -31,10 +31,11 @@ public interface WorkflowLib {
* Parse a workflow application definition.
*
* @param wfXml string containing the workflow definition.
+ * @param jobConf job configuration
* @return the parse workflow application.
* @throws WorkflowException thrown if the definition could not be parsed.
*/
- public WorkflowApp parseDef(String wfXml) throws WorkflowException;
+ public WorkflowApp parseDef(String wfXml, Configuration jobConf) throws WorkflowException;
/**
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowAppParser.java Tue Aug 7 04:39:50 2012
@@ -21,9 +21,12 @@ import org.apache.oozie.workflow.Workflo
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XmlUtils;
import org.apache.oozie.util.ParamChecker;
+import org.apache.oozie.util.ParameterVerifier;
+import org.apache.oozie.util.ParameterVerifierException;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.ActionService;
+import org.apache.hadoop.conf.Configuration;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
@@ -58,6 +61,7 @@ public class LiteWorkflowAppParser {
private static final String SLA_INFO = "info";
private static final String CREDENTIALS = "credentials";
private static final String GLOBAL = "global";
+ private static final String PARAMETERS = "parameters";
private static final String NAME_A = "name";
private static final String CRED_A = "cred";
@@ -107,7 +111,7 @@ public class LiteWorkflowAppParser {
* @return LiteWorkflowApp
* @throws WorkflowException
*/
- public LiteWorkflowApp validateAndParse(Reader reader) throws WorkflowException {
+ public LiteWorkflowApp validateAndParse(Reader reader, Configuration jobConf) throws WorkflowException {
try {
StringWriter writer = new StringWriter();
IOUtils.copyCharStream(reader, writer);
@@ -119,6 +123,7 @@ public class LiteWorkflowAppParser {
}
Element wfDefElement = XmlUtils.parseXml(strDef);
+ ParameterVerifier.verifyParameters(jobConf, wfDefElement);
LiteWorkflowApp app = parse(strDef, wfDefElement);
Map<String, VisitStatus> traversed = new HashMap<String, VisitStatus>();
traversed.put(app.getNode(StartNodeDef.START).getName(), VisitStatus.VISITING);
@@ -129,6 +134,9 @@ public class LiteWorkflowAppParser {
}
return app;
}
+ catch (ParameterVerifierException ex) {
+ throw new WorkflowException(ex);
+ }
catch (JDOMException ex) {
throw new WorkflowException(ErrorCode.E0700, ex.getMessage(), ex);
}
@@ -341,7 +349,12 @@ public class LiteWorkflowAppParser {
global = eNode;
}
else {
- throw new WorkflowException(ErrorCode.E0703, eNode.getName());
+ if (eNode.getName().equals(PARAMETERS)) {
+ // No operation is required
+ }
+ else {
+ throw new WorkflowException(ErrorCode.E0703, eNode.getName());
+ }
}
}
}
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowLib.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowLib.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowLib.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/workflow/lite/LiteWorkflowLib.java Tue Aug 7 04:39:50 2012
@@ -49,10 +49,10 @@ public abstract class LiteWorkflowLib im
}
@Override
- public WorkflowApp parseDef(String appXml) throws WorkflowException {
+ public WorkflowApp parseDef(String appXml, Configuration jobConf) throws WorkflowException {
ParamChecker.notEmpty(appXml, "appXml");
return new LiteWorkflowAppParser(schema, controlHandlerClass, decisionHandlerClass, actionHandlerClass)
- .validateAndParse(new StringReader(appXml));
+ .validateAndParse(new StringReader(appXml), jobConf);
}
@Override
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/wf/TestSubmitMRXCommand.java Tue Aug 7 04:39:50 2012
@@ -67,7 +67,7 @@ public class TestSubmitMRXCommand extend
//verifying is a valid WF
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
- wps.parseDef(xml);
+ wps.parseDef(xml, conf);
Element wfE = XmlUtils.parseXml(xml);
Namespace ns = wfE.getNamespace();
Added: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParameterVerifier.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParameterVerifier.java?rev=1370117&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParameterVerifier.java (added)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParameterVerifier.java Tue Aug 7 04:39:50 2012
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.ErrorCode;
+import org.apache.oozie.test.XTestCase;
+
+public class TestParameterVerifier extends XTestCase {
+
+ public void testVerifyParametersNull() throws Exception {
+ try {
+ ParameterVerifier.verifyParameters(null, XmlUtils.parseXml("<root xmlns=\"uri:oozie:workflow:0.4\"></root>"));
+ fail();
+ } catch (IllegalArgumentException ex) {
+ assertEquals("conf cannot be null", ex.getMessage());
+ }
+
+ Configuration conf = new Configuration(false);
+ conf.set("A", "a");
+ ParameterVerifier.verifyParameters(conf, null);
+ assertEquals(1, conf.size());
+ assertEquals("a", conf.get("A"));
+
+ try {
+ ParameterVerifier.verifyParameters(null, null);
+ } catch (IllegalArgumentException ex) {
+ assertEquals("conf cannot be null", ex.getMessage());
+ }
+ }
+
+ public void testVerifyParametersEmpty() throws Exception {
+ Configuration conf = new Configuration(false);
+ conf.set("A", "a");
+
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml("<root xmlns=\"uri:oozie:workflow:0.4\"></root>"));
+ assertEquals(1, conf.size());
+ assertEquals("a", conf.get("A"));
+
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml("<root xmlns=\"uri:oozie:workflow:0.4\">"
+ + "<parameters></parameters></root>"));
+ assertEquals(1, conf.size());
+ assertEquals("a", conf.get("A"));
+ }
+
+ public void testVerifyParametersMissing() throws Exception {
+ Configuration conf = new Configuration(false);
+
+ String str = "<root xmlns=\"uri:oozie:workflow:0.4\"><parameters>"
+ + "<property><name>hello</name></property>"
+ + "</parameters></root>";
+ try {
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(str));
+ fail();
+ } catch(ParameterVerifierException ex) {
+ assertEquals(ErrorCode.E0738, ex.getErrorCode());
+ assertTrue(ex.getMessage().endsWith("hello"));
+ assertTrue(ex.getMessage().contains("1"));
+ assertEquals(0, conf.size());
+ }
+
+ conf = new Configuration(false);
+
+ str = "<root xmlns=\"uri:oozie:workflow:0.4\"><parameters>"
+ + "<property><name>hello</name><value>world</value></property>"
+ + "</parameters></root>";
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(str));
+ assertEquals(1, conf.size());
+ assertEquals("world", conf.get("hello"));
+
+ conf = new Configuration(false);
+
+ str = "<root xmlns=\"uri:oozie:workflow:0.4\"><parameters>"
+ + "<property><name>hello</name></property>"
+ + "<property><name>foo</name><value>bar</value></property>"
+ + "<property><name>meh</name></property>"
+ + "</parameters></root>";
+ try {
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(str));
+ fail();
+ } catch(ParameterVerifierException ex) {
+ assertEquals(ErrorCode.E0738, ex.getErrorCode());
+ assertTrue(ex.getMessage().endsWith("hello, meh"));
+ assertFalse(ex.getMessage().contains("foo"));
+ assertTrue(ex.getMessage().contains("2"));
+ assertEquals(1, conf.size());
+ assertEquals("bar", conf.get("foo"));
+ }
+ }
+
+ public void testVerifyParametersDefined() throws Exception {
+ Configuration conf = new Configuration(false);
+ conf.set("hello", "planet");
+
+ String str = "<root xmlns=\"uri:oozie:workflow:0.4\"><parameters>"
+ + "<property><name>hello</name></property>"
+ + "</parameters></root>";
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(str));
+ assertEquals(1, conf.size());
+ assertEquals("planet", conf.get("hello"));
+
+ str = "<root xmlns=\"uri:oozie:workflow:0.4\"><parameters>"
+ + "<property><name>hello</name><value>world</value></property>"
+ + "</parameters></root>";
+ ParameterVerifier.verifyParameters(conf, XmlUtils.parseXml(str));
+ assertEquals(1, conf.size());
+ assertEquals("planet", conf.get("hello"));
+ }
+}
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java Tue Aug 7 04:39:50 2012
@@ -41,6 +41,7 @@ import org.apache.oozie.util.IOUtils;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.action.hadoop.DistcpActionExecutor;
import org.apache.oozie.action.hadoop.HiveActionExecutor;
+import org.apache.hadoop.conf.Configuration;
public class TestLiteWorkflowAppParser extends XTestCase {
public static String dummyConf = "<java></java>";
@@ -65,7 +66,8 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1));
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1),
+ new Configuration());
String d = app.getNode("d").getConf();
String expectedD =
@@ -104,7 +106,8 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1));
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global.xml", -1),
+ new Configuration());
String e = app.getNode("e").getConf();
String expectedE =
@@ -140,7 +143,8 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1));
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1),
+ new Configuration());
String a = app.getNode("a").getConf();
String expectedA =
@@ -179,7 +183,8 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1));
+ LiteWorkflowApp app = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext.xml", -1),
+ new Configuration());
String b = app.getNode("b").getConf();
String expectedB =
@@ -215,10 +220,11 @@ public class TestLiteWorkflowAppParser e
// If no global section is defined, some extension actions (e.g. hive) must still have name-node and job-tracker elements
// or the handleGlobal() method will throw an exception
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext-no-global.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid-global-ext-no-global.xml", -1), new Configuration());
try {
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-invalid-global-ext-no-global.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-invalid-global-ext-no-global.xml", -1),
+ new Configuration());
fail();
}
catch (WorkflowException ex) {
@@ -235,10 +241,10 @@ public class TestLiteWorkflowAppParser e
LiteWorkflowStoreService.LiteDecisionHandler.class,
LiteWorkflowStoreService.LiteActionHandler.class);
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-valid.xml", -1), new Configuration());
try {
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-loop1-invalid.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-loop1-invalid.xml", -1), new Configuration());
fail();
}
catch (WorkflowException ex) {
@@ -249,7 +255,7 @@ public class TestLiteWorkflowAppParser e
}
try {
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-unsupported-action.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-unsupported-action.xml", -1), new Configuration());
fail();
}
catch (WorkflowException ex) {
@@ -260,7 +266,7 @@ public class TestLiteWorkflowAppParser e
}
try {
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-loop2-invalid.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-loop2-invalid.xml", -1), new Configuration());
fail();
}
catch (WorkflowException ex) {
@@ -271,7 +277,7 @@ public class TestLiteWorkflowAppParser e
}
try {
- parser.validateAndParse(IOUtils.getResourceAsReader("wf-transition-invalid.xml", -1));
+ parser.validateAndParse(IOUtils.getResourceAsReader("wf-transition-invalid.xml", -1), new Configuration());
fail();
}
catch (WorkflowException ex) {
@@ -293,7 +299,7 @@ public class TestLiteWorkflowAppParser e
public void run() {
try {
// Validate against wf def
- parser.validateAndParse(new StringReader(TestSchemaService.APP_NEG_TEST));
+ parser.validateAndParse(new StringReader(TestSchemaService.APP_NEG_TEST), new Configuration());
fail("Expected to catch WorkflowException but didn't encounter any");
} catch (WorkflowException we) {
assertEquals(ErrorCode.E0701, we.getErrorCode());
Modified: incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki Tue Aug 7 04:39:50 2012
@@ -12,6 +12,11 @@ The goal of this document is to define a
---++ Changelog
+---+++!! 26/JUL/2012
+
+ * #Appendix A, updated XML schema 0.4 to include =parameters= element
+ * #6.5 Updated to mention about =parameters= element as of schema 0.4
+
---+++!! 23/NOV/2011:
* Update execution order typo
@@ -1176,6 +1181,53 @@ In the above example there are 6 configu
IMPORTANT: Note that this example is not completely correct as it always consumes the last 24 instances of the 'logs' dataset. It is assumed that all days have 24 hours. For timezones that observe daylight saving this application will not work as expected as it will consume the wrong number of dataset instances in DST switch days. To be able to handle these scenarios, the =${coord:hoursInDays(int n)}= and =${coord:daysInMonths(int n)}= EL functions must be used (refer to section #6.6.2 and #6.6.3).
+If the above 6 properties are not specified, the job will fail.
+
+As of schema 0.4, a list of formal parameters can be provided which will allow Oozie to verify, at submission time, that said
+properties are actually specified (i.e. before the job is executed and fails). Default values can also be provided.
+
+*Example:*
+
+The previous parameterized coordinator application definition with formal parameters:
+
+<verbatim>
+ <coordinator-app name="app-coord" frequency="${coord:days(1)}"
+ start="${jobStart}" end="${jobEnd}" timezone="${timezone}"
+ xmlns="uri:oozie:coordinator:0.1">
+ <parameters>
+ <property>
+ <name>jobStart</name>
+ </property>
+ <property>
+ <name>jobEnd</name>
+ <value>2012-12-01T22:00Z</value>
+ </property>
+ </parameters>
+ <datasets>
+ <dataset name="logs" frequency="${coord:hours(1)}"
+ initial-instance="${logsInitialInstance}" timezone="${timezone}">
+ <uri-template>
+ hdfs://bar:9000/app/logs/${market}/${language}/${YEAR}${MONTH}/${DAY}/${HOUR}
+ </uri-template>
+ </dataset>
+ </datasets>
+ <input-events>
+ <data-in name="input" dataset="logs">
+ <start-instance>${coord:current(-23)}</start-instance>
+ <end-instance>${coord:current(0)}</end-instance>
+ </data-in>
+ </input-events>
+ <action>
+ <workflow>
+ ...
+ </workflow>
+ </action>
+ </coordinator-app>
+</verbatim>
+
+In the above example, if =jobStart= is not specified, Oozie will print an error message instead of submitting the job. If
+=jobEnd= is not specified, Oozie will use the default value, =2012-12-01T22:00Z=.
+
---+++ 6.6. Parameterization of Dataset Instances in Input and Output Events
A coordinator application job typically launches several coordinator actions during its lifetime. A coordinator action typically uses its creation (materialization) time to resolve the specific datasets instances required for its input and output events.
@@ -2691,6 +2743,131 @@ Refer to the [[DG_CoordinatorRerun][Reru
</xs:schema>
</verbatim>
+---++++ Oozie Coordinator Schema 0.4
+
+<verbatim>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:coordinator="uri:oozie:coordinator:0.2"
+ elementFormDefault="qualified" targetNamespace="uri:oozie:coordinator:0.2">
+
+ <xs:element name="coordinator-app" type="coordinator:COORDINATOR-APP"/>
+ <xs:element name="datasets" type="coordinator:DATASETS"/>
+ <xs:simpleType name="IDENTIFIER">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="([a-zA-Z]([\-_a-zA-Z0-9])*){1,39})"/>
+ </xs:restriction>
+ </xs:simpleType>
+ <xs:complexType name="COORDINATOR-APP">
+ <xs:sequence>
+ <xs:element name="parameters" type="coordinator:PARAMETERS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="controls" type="coordinator:CONTROLS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="datasets" type="coordinator:DATASETS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="input-events" type="coordinator:INPUTEVENTS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="output-events" type="coordinator:OUTPUTEVENTS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="action" type="coordinator:ACTION" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="frequency" type="xs:string" use="required"/>
+ <xs:attribute name="start" type="xs:string" use="required"/>
+ <xs:attribute name="end" type="xs:string" use="required"/>
+ <xs:attribute name="timezone" type="xs:string" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="PARAMETERS">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="CONTROLS">
+ <xs:sequence minOccurs="0" maxOccurs="1">
+ <xs:element name="timeout" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="concurrency" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="execution" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="throttle" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="DATASETS">
+ <xs:sequence minOccurs="0" maxOccurs="1">
+ <xs:element name="include" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:choice minOccurs="0" maxOccurs="unbounded">
+ <xs:element name="dataset" type="coordinator:SYNCDATASET" minOccurs="0" maxOccurs="1"/>
+ </xs:choice>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="SYNCDATASET">
+ <xs:sequence>
+ <xs:element name="uri-template" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="done-flag" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="frequency" type="xs:string" use="required"/>
+ <xs:attribute name="initial-instance" type="xs:string" use="required"/>
+ <xs:attribute name="timezone" type="xs:string" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="INPUTEVENTS">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="data-in" type="coordinator:DATAIN" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="DATAIN">
+ <xs:choice minOccurs="1" maxOccurs="1">
+ <xs:element name="instance" type="xs:string" minOccurs="1" maxOccurs="unbounded"/>
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="start-instance" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="end-instance" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:choice>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="dataset" type="coordinator:IDENTIFIER" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="OUTPUTEVENTS">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="data-out" type="coordinator:DATAOUT" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="DATAOUT">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="instance" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="coordinator:IDENTIFIER" use="required"/>
+ <xs:attribute name="dataset" type="coordinator:IDENTIFIER" use="required"/>
+ </xs:complexType>
+ <xs:complexType name="ACTION">
+ <xs:sequence minOccurs="1" maxOccurs="1">
+ <xs:element name="workflow" type="coordinator:WORKFLOW" minOccurs="1" maxOccurs="1"/>
+ <xs:any namespace="uri:oozie:sla:0.1" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:complexType name="WORKFLOW">
+ <xs:sequence>
+ <xs:element name="app-path" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="configuration" type="coordinator:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="FLAG"/>
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+</xs:schema>
+</verbatim>
+
---++++ Oozie SLA Schema
<verbatim>
<?xml version="1.0" encoding="UTF-8"?>
Modified: incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/WorkflowFunctionalSpec.twiki Tue Aug 7 04:39:50 2012
@@ -14,6 +14,10 @@ Map/Reduce and Pig jobs.
%TOC%
---++ Changelog
+---+++!! 2012JUL26
+
+ * #Appendix A, updated XML schema 0.4 to include =parameters= element
+ * #4.1 Updated to mention about =parameters= element as of schema 0.4
---+++!! 2012JUL23
* #Appendix A, updated XML schema 0.4 (Fs action)
@@ -1465,6 +1469,60 @@ When submitting a workflow job for the w
* =inputDir:=
* =outputDir:=
+If the above 3 properties are not specified, the job will fail.
+
+As of schema 0.4, a list of formal parameters can be provided which will allow Oozie to verify, at submission time, that said
+properties are actually specified (i.e. before the job is executed and fails). Default values can also be provided.
+
+*Example:*
+
+The previous parameterized workflow definition with formal parameters:
+
+<verbatim>
+<workflow-app name='hello-wf' xmlns="uri:oozie:workflow:0.4">
+ <parameters>
+ <property>
+ <name>inputDir</name>
+ </property>
+ <property>
+ <name>outputDir</name>
+ <value>out-dir</value>
+ </property>
+ </parameters>
+ ...
+ <action name='firstjob'>
+ <map-reduce>
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <configuration>
+ <property>
+ <name>mapred.mapper.class</name>
+ <value>com.foo.FirstMapper</value>
+ </property>
+ <property>
+ <name>mapred.reducer.class</name>
+ <value>com.foo.FirstReducer</value>
+ </property>
+ <property>
+ <name>mapred.input.dir</name>
+ <value>${inputDir}</value>
+ </property>
+ <property>
+ <name>mapred.output.dir</name>
+ <value>${outputDir}</value>
+ </property>
+ </configuration>
+ </map-reduce>
+ <ok to='secondjob'/>
+ <error to='killcleanup'/>
+ </action>
+ ...
+</workflow-app>
+</verbatim>
+
+In the above example, if =inputDir= is not specified, Oozie will print an error message instead of submitting the job. If
+=outputDir= is not specified, Oozie will use the default value, =out-dir=.
+
#WorkflowELSupport
---+++ 4.2 Expression Language Functions
@@ -2236,7 +2294,8 @@ Example of a global element:
<xs:complexType name="WORKFLOW-APP">
<xs:sequence>
- <xs:element name="credentials" type="workflow:CREDENTIALS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="parameters" type="workflow:PARAMETERS" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="credentials" type="workflow:CREDENTIALS" minOccurs="0" maxOccurs="1"/>
<xs:element name="start" type="workflow:START" minOccurs="1" maxOccurs="1"/>
<xs:choice minOccurs="0" maxOccurs="unbounded">
<xs:element name="decision" type="workflow:DECISION" minOccurs="1" maxOccurs="1"/>
@@ -2251,6 +2310,20 @@ Example of a global element:
<xs:attribute name="name" type="xs:string" use="required"/>
</xs:complexType>
+ <xs:complexType name="PARAMETERS">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
<xs:complexType name="START">
<xs:attribute name="to" type="workflow:IDENTIFIER" use="required"/>
</xs:complexType>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1370117&r1=1370116&r2=1370117&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Tue Aug 7 04:39:50 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-239 Add formal parameters to WF & COORD XML (rkanter via tucu)
OOZIE-938 Remove some duplicated code in FsActionExecutor (rkanter via tucu)
OOZIE-939 JT_PRINCIPAL and NN_PRINCIPAL must be added back to XOozieClient (rkanter via tucu)
OOZIE-930 Bundle not pass SUSPEND command to PAUSED coord job (virag)