You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/05/08 10:40:42 UTC
[36/37] oozie git commit: OOZIE-2873 Check El Functions before
submitting the coordinator (satishsaley)
OOZIE-2873 Check El Functions before submitting the coordinator (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f669852b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f669852b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f669852b
Branch: refs/heads/oya
Commit: f669852b332af78aaf658f69e785fad31a1aa392
Parents: 87ed1b3
Author: satishsaley <sa...@apache.org>
Authored: Wed May 3 20:17:44 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Wed May 3 20:17:44 2017 -0700
----------------------------------------------------------------------
.../command/coord/CoordSubmitXCommand.java | 25 +++++++++
.../command/coord/TestCoordSubmitXCommand.java | 30 ++++++++++
.../resources/coord-invalid-el-function.xml | 35 ++++++++++++
.../resources/coord-invalid-output-instance.xml | 58 ++++++++++++++++++++
.../coord-multiple-input-instance4.xml | 2 +-
.../coord-multiple-input-start-instance2.xml | 2 +-
.../coord-multiple-output-instance4.xml | 2 +-
release-log.txt | 1 +
8 files changed, 152 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
index 969336d..0c50621 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
@@ -245,6 +245,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
LogUtils.setLogInfo(coordJob);
if (!dryrun) {
+ verifyElFunctions(coordJob);
queueMaterializeTransitionXCommand(jobId);
}
else {
@@ -1348,4 +1349,28 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
@Override
public void performWrites() throws CommandException {
}
+
+ /**
+ * Verify el functions using dryrun. Throws an exception if dry run throws
+ * an exception and marks the coordJob as FAILED.
+ *
+ * @param coordJob
+ * @throws Exception
+ */
+ private void verifyElFunctions(CoordinatorJobBean coordJob) throws Exception {
+ Job.Status prevStatus = coordJob.getStatus();
+ try {
+ // dry run will catch issues with el functions if any
+ getDryRun(coordJob);
+ }
+ catch (Exception e) {
+ prevStatus = CoordinatorJob.Status.FAILED;
+ throw e;
+ }
+ finally {
+ coordJob.setStatus(prevStatus);
+ coordJob.resetPending();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
index dcc97ad..91b3792 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
@@ -20,6 +20,7 @@ package org.apache.oozie.command.coord;
import java.io.File;
import java.io.FileWriter;
+import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.net.URI;
@@ -1569,4 +1570,33 @@ public class TestCoordSubmitXCommand extends XDataTestCase {
.contains("tmp/coord/b/2009/03"));
}
+ public void testFailureOnSubmitWithInvalidInstance() throws Exception {
+ testFailure("coord-invalid-output-instance.xml");
+ }
+
+ public void testFailureOnSubmitWithInvalidElFunction() throws Exception {
+ testFailure("coord-invalid-el-function.xml");
+ }
+
+ private void testFailure(String coordinatorXml) throws IOException {
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+
+ Reader reader = IOUtils.getResourceAsReader(coordinatorXml, -1);
+ Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
+ IOUtils.copyCharStream(reader, writer);
+
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+
+ try {
+ sc.call();
+ fail("Should throw an exception");
+ }
+ catch (CommandException e) {
+ assertEquals(Job.Status.FAILED, sc.getJob().getStatus());
+ assertEquals(e.getErrorCode(), ErrorCode.E0803);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-invalid-el-function.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-invalid-el-function.xml b/core/src/test/resources/coord-invalid-el-function.xml
new file mode 100644
index 0000000..11dc478
--- /dev/null
+++ b/core/src/test/resources/coord-invalid-el-function.xml
@@ -0,0 +1,35 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}"
+ start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC">
+ <controls>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ <property>
+ <name>yyyymmdd</name>
+ <value>${coord:formatTime(coord:nominalTime(), 'DAY')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-invalid-output-instance.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-invalid-output-instance.xml b/core/src/test/resources/coord-invalid-output-instance.xml
new file mode 100644
index 0000000..a90706e
--- /dev/null
+++ b/core/src/test/resources/coord-invalid-output-instance.xml
@@ -0,0 +1,58 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}"
+ start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC">
+ <controls>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <datasets>
+ <dataset name="a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
+ </dataset>
+ <dataset name="local_a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
+ </dataset>
+ </datasets>
+ <input-events>
+ <data-in name="A" dataset="a">
+ <instance>${coord:future(0, 1)}</instance>
+ </data-in>
+ </input-events>
+ <!-- Invalid coordinator application coord:latest()/future(), not permitted in output-events -->
+ <output-events>
+ <data-out name="LOCAL_A" dataset="local_a">
+ <instance>${coord:future(0,2)}</instance>
+ </data-out>
+ </output-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ <property>
+ <name>inputA</name>
+ <value>${coord:dataIn('A')}</value>
+ </property>
+ <property>
+ <name>inputB</name>
+ <value>${coord:dataOut('LOCAL_A')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-multiple-input-instance4.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-input-instance4.xml b/core/src/test/resources/coord-multiple-input-instance4.xml
index 023df26..e092465 100644
--- a/core/src/test/resources/coord-multiple-input-instance4.xml
+++ b/core/src/test/resources/coord-multiple-input-instance4.xml
@@ -35,7 +35,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:future(0,2)}</instance>
+ <instance>${coord:current(0)}</instance>
</data-out>
</output-events>
<action>
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-multiple-input-start-instance2.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-input-start-instance2.xml b/core/src/test/resources/coord-multiple-input-start-instance2.xml
index 83d3773..6865d39 100644
--- a/core/src/test/resources/coord-multiple-input-start-instance2.xml
+++ b/core/src/test/resources/coord-multiple-input-start-instance2.xml
@@ -36,7 +36,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:future(0,2)}</instance>
+ <instance>${coord:current(0)}</instance>
</data-out>
</output-events>
<action>
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-multiple-output-instance4.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-output-instance4.xml b/core/src/test/resources/coord-multiple-output-instance4.xml
index 32d0786..9ed4e40 100644
--- a/core/src/test/resources/coord-multiple-output-instance4.xml
+++ b/core/src/test/resources/coord-multiple-output-instance4.xml
@@ -35,7 +35,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:formatTime(coord:current(0),'yyyy-MM-dd')}</instance>
+ <instance>${coord:current(0)}</instance>
</data-out>
</output-events>
<action>
http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d998b2d..944e623 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2873 Check El Functions before submitting the coordinator (satishsaley)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
OOZIE-2827 amend More directly view of the coordinator’s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
OOZIE-2851 spelling mistakes in examples (Artem Ervits via gezapeti)