You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/03/14 23:53:48 UTC
svn commit: r1300773 - in /incubator/oozie/trunk: ./
core/src/main/java/org/apache/oozie/command/coord/
core/src/main/java/org/apache/oozie/util/
core/src/test/java/org/apache/oozie/command/coord/
core/src/test/java/org/apache/oozie/util/ core/src/test...
Author: tucu
Date: Wed Mar 14 22:53:48 2012
New Revision: 1300773
URL: http://svn.apache.org/viewvc?rev=1300773&view=rev
Log:
OOZIE-675 checkMultipleTimeInstances doesn't work for EL extensions. EX: (sriksun via tucu)
Added:
incubator/oozie/trunk/core/src/test/resources/coord-multiple-input-instance4.xml
incubator/oozie/trunk/core/src/test/resources/coord-multiple-output-instance4.xml
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ELEvaluator.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java?rev=1300773&r1=1300772&r2=1300773&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java Wed Mar 14 22:53:48 2012
@@ -139,6 +139,7 @@ public class CoordSubmitXCommand extends
private ELEvaluator evalNofuncs = null;
private ELEvaluator evalData = null;
private ELEvaluator evalInst = null;
+ private ELEvaluator evalAction = null;
private ELEvaluator evalSla = null;
static {
@@ -315,21 +316,44 @@ public class CoordSubmitXCommand extends
throw new CoordinatorJobException(ErrorCode.E1021, "<instance> tag within " + eventType + " is empty!");
}
instanceValue = instance.getContent(0).toString();
- if (instanceValue.contains(",")) { // reaching this block implies instance is not empty i.e. length > 0
- String correctAction = null;
- if(dataType.equals(COORD_INPUT_EVENTS_DATA_IN)) {
- correctAction = "Coordinator app definition should have separate <instance> tag per data-in instance";
- } else if(dataType.equals(COORD_OUTPUT_EVENTS_DATA_OUT)) {
- correctAction = "Coordinator app definition can have only one <instance> tag per data-out instance";
- }
- throw new CoordinatorJobException(ErrorCode.E1021, eventType + " instance '" + instanceValue
- + "' contains more than one date instance. Coordinator job NOT SUBMITTED. " + correctAction);
+ boolean isInvalid = false;
+ try {
+ isInvalid = evalAction.checkForExistence(instanceValue, ",");
+ } catch (Exception e) {
+ handleELParseException(eventType, dataType, instanceValue);
+ }
+ if (isInvalid) { // reaching this block implies instance is not empty i.e. length > 0
+ handleExpresionWithMultipleInstances(eventType, dataType, instanceValue);
}
}
}
}
}
+ private void handleELParseException(String eventType, String dataType, String instanceValue)
+ throws CoordinatorJobException {
+ String correctAction = null;
+ if(dataType.equals(COORD_INPUT_EVENTS_DATA_IN)) {
+ correctAction = "Coordinator app definition should have valid <instance> tag for data-in";
+ } else if(dataType.equals(COORD_OUTPUT_EVENTS_DATA_OUT)) {
+ correctAction = "Coordinator app definition should have valid <instance> tag for data-out";
+ }
+ throw new CoordinatorJobException(ErrorCode.E1021, eventType + " instance '" + instanceValue
+ + "' is not valid. Coordinator job NOT SUBMITTED. " + correctAction);
+ }
+
+ private void handleExpresionWithMultipleInstances(String eventType, String dataType, String instanceValue)
+ throws CoordinatorJobException {
+ String correctAction = null;
+ if(dataType.equals(COORD_INPUT_EVENTS_DATA_IN)) {
+ correctAction = "Coordinator app definition should have separate <instance> tag per data-in instance";
+ } else if(dataType.equals(COORD_OUTPUT_EVENTS_DATA_OUT)) {
+ correctAction = "Coordinator app definition can have only one <instance> tag per data-out instance";
+ }
+ throw new CoordinatorJobException(ErrorCode.E1021, eventType + " instance '" + instanceValue
+ + "' contains more than one date instance. Coordinator job NOT SUBMITTED. " + correctAction);
+ }
+
/**
* Read the application XML and validate against coordinator Schema
*
@@ -514,6 +538,7 @@ public class CoordSubmitXCommand extends
evalNofuncs = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-nofuncs");
evalInst = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-instances");
evalSla = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-sla-submit");
+ evalAction = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-action-start");
}
/**
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ELEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ELEvaluator.java?rev=1300773&r1=1300772&r2=1300773&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ELEvaluator.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/util/ELEvaluator.java Wed Mar 14 22:53:48 2012
@@ -18,6 +18,7 @@
package org.apache.oozie.util;
import org.apache.commons.el.ExpressionEvaluatorImpl;
+import org.apache.commons.el.ExpressionString;
import javax.servlet.jsp.el.ELException;
import javax.servlet.jsp.el.ExpressionEvaluator;
@@ -138,7 +139,7 @@ public class ELEvaluator {
private Context context;
- private ExpressionEvaluator evaluator = new ExpressionEvaluatorImpl();
+ private ExpressionEvaluatorImpl evaluator = new ExpressionEvaluatorImpl();
/**
* Creates an ELEvaluator with no functions and no variables defined.
@@ -214,4 +215,41 @@ public class ELEvaluator {
}
}
+ /**
+ * Check if the input expression contains sequence statically. for example
+ * identify if "," is present outside of a function invocation in the given
+ * expression. Ex "${func('abc')},${func('def'}",
+ *
+ * @param expr - Expression string
+ * @param sequence - char sequence to check in the input expression
+ * @return true if present
+ * @throws Exception Exception thrown if an EL function failed due to a
+ * transient error or EL expression could not be parsed
+ */
+ public boolean checkForExistence(String expr, String sequence)
+ throws Exception {
+ try {
+ Object exprString = evaluator.parseExpressionString(expr);
+ if (exprString instanceof ExpressionString) {
+ for (Object element : ((ExpressionString)exprString).getElements()) {
+ if (element instanceof String &&
+ element.toString().contains(sequence)) {
+ return true;
+ }
+ }
+ } else if (exprString instanceof String) {
+ if (((String)exprString).contains(sequence)) {
+ return true;
+ }
+ }
+ return false;
+ } catch (ELException ex) {
+ if (ex.getRootCause() instanceof Exception) {
+ throw (Exception) ex.getRootCause();
+ }
+ else {
+ throw ex;
+ }
+ }
+ }
}
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java?rev=1300773&r1=1300772&r2=1300773&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java Wed Mar 14 22:53:48 2012
@@ -149,13 +149,26 @@ public class TestCoordSubmitXCommand ext
catch (CommandException e) {
fail("Unexpected failure: " + e);
}
+
+ // CASE 4: Success case i.e. Single instances for input and single instance for output, but both with ","
+ reader = IOUtils.getResourceAsReader("coord-multiple-input-instance4.xml", -1);
+ writer = new FileWriter(appPath);
+ IOUtils.copyCharStream(reader, writer);
+ sc = new CoordSubmitXCommand(conf, "UNIT_TESTING");
+
+ try {
+ sc.call();
+ }
+ catch (CommandException e) {
+ fail("Unexpected failure: " + e);
+ }
}
/**
* Testing for when user tries to submit a coordinator application having data-out events
* that erroneously specify multiple output data instances inside a single <instance> tag.
* Job gives submission error and indicates appropriate correction
- * Testing negative(error) cases. Positive(success) case is covered in another test case "testBasicSubmit".
+ * Testing negative(error) cases as well as Positive(success) cases.
*/
public void testBasicSubmitWithMultipleInstancesOutputEvent() throws Exception {
Configuration conf = new XConfiguration();
@@ -211,6 +224,19 @@ public class TestCoordSubmitXCommand ext
assertEquals(e.getErrorCode(), ErrorCode.E0701);
assertTrue(e.getMessage().contains("No child element is expected at this point"));
}
+
+ // CASE 4: Success case, where only one instance is configured, but expression has a ","
+ reader = IOUtils.getResourceAsReader("coord-multiple-output-instance4.xml", -1);
+ writer = new FileWriter(appPath);
+ IOUtils.copyCharStream(reader, writer);
+ sc = new CoordSubmitXCommand(conf, "UNIT_TESTING");
+
+ try {
+ sc.call();
+ }
+ catch (CommandException e) {
+ fail("Not expected to fail here");
+ }
}
/**
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java?rev=1300773&r1=1300772&r2=1300773&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestELEvaluator.java Wed Mar 14 22:53:48 2012
@@ -39,6 +39,10 @@ public class TestELEvaluator extends XTe
return "c";
}
+ public static String functionD(String in1, String in2) {
+ return in1 + "::" + in2;
+ }
+
public static String functionError() throws ELEvaluationException {
throw new ELEvaluationException("m", null);
}
@@ -46,6 +50,7 @@ public class TestELEvaluator extends XTe
private static Method functionA;
private static Method functionB;
private static Method functionC;
+ private static Method functionD;
private static Method functionError;
static {
@@ -53,6 +58,8 @@ public class TestELEvaluator extends XTe
functionA = TestELEvaluator.class.getMethod("functionA");
functionB = TestELEvaluator.class.getMethod("functionB");
functionC = TestELEvaluator.class.getDeclaredMethod("functionC");
+ functionD = TestELEvaluator.class.getDeclaredMethod("functionD",
+ String.class, String.class);
functionError = TestELEvaluator.class.getDeclaredMethod("functionError");
}
catch (Exception ex) {
@@ -167,4 +174,28 @@ public class TestELEvaluator extends XTe
}
}
+ public void testCheckForExistence() throws Exception {
+ ELEvaluator.Context support = new ELEvaluator.Context();
+ support.setVariable("a", "A");
+ support.addFunction("a", "a", functionA);
+ support.addFunction("a", "d", functionD);
+ ELEvaluator evaluator = new ELEvaluator(support);
+ assertNull(ELEvaluator.getCurrent());
+ assertEquals("a", evaluator.evaluate("${a:a()}", String.class));
+ assertEquals("a,a", evaluator.evaluate("${a:a()},${a:a()}", String.class));
+ try {
+ evaluator.evaluate("${a:a(), a:a()}", String.class);
+ fail("Evaluated bad expression");
+ } catch (ELException ignore) { }
+ assertTrue(evaluator.checkForExistence("${a:a()}${a:a()}!", "!"));
+ assertTrue(evaluator.checkForExistence("${a:a()},${a:a()}", ","));
+ assertFalse(evaluator.checkForExistence("${a:d('foo', 'bar')}", ","));
+ try {
+ evaluator.checkForExistence("${a:a(), a:a()}", ",");
+ fail("Parsed bad expression");
+ } catch (ELException ignore) { }
+
+ assertNull(ELEvaluator.getCurrent());
+ }
+
}
Added: incubator/oozie/trunk/core/src/test/resources/coord-multiple-input-instance4.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/resources/coord-multiple-input-instance4.xml?rev=1300773&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/test/resources/coord-multiple-input-instance4.xml (added)
+++ incubator/oozie/trunk/core/src/test/resources/coord-multiple-input-instance4.xml Wed Mar 14 22:53:48 2012
@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}" start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC">
+ <controls>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <datasets>
+ <dataset name="a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
+ </dataset>
+ <dataset name="local_a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
+ </dataset>
+ </datasets>
+ <input-events>
+ <data-in name="A" dataset="a">
+ <instance>${coord:future(0, 1)}</instance>
+ </data-in>
+ </input-events>
+ <output-events>
+ <data-out name="LOCAL_A" dataset="local_a">
+ <instance>${coord:future(0,2)}</instance>
+ </data-out>
+ </output-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ <property>
+ <name>inputA</name>
+ <value>${coord:dataIn('A')}</value>
+ </property>
+ <property>
+ <name>inputB</name>
+ <value>${coord:dataOut('LOCAL_A')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
\ No newline at end of file
Added: incubator/oozie/trunk/core/src/test/resources/coord-multiple-output-instance4.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/resources/coord-multiple-output-instance4.xml?rev=1300773&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/test/resources/coord-multiple-output-instance4.xml (added)
+++ incubator/oozie/trunk/core/src/test/resources/coord-multiple-output-instance4.xml Wed Mar 14 22:53:48 2012
@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}" start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC">
+ <controls>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <datasets>
+ <dataset name="a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
+ </dataset>
+ <dataset name="local_a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
+ <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
+ </dataset>
+ </datasets>
+ <input-events>
+ <data-in name="A" dataset="a">
+ <instance>${coord:future(0,2)}</instance>
+ </data-in>
+ </input-events>
+ <output-events>
+ <data-out name="LOCAL_A" dataset="local_a">
+ <instance>${coord:formatTime(coord:current(0),'yyyy-MM-dd')}</instance>
+ </data-out>
+ </output-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ <property>
+ <name>inputA</name>
+ <value>${coord:dataIn('A')}</value>
+ </property>
+ <property>
+ <name>inputB</name>
+ <value>${coord:dataOut('LOCAL_A')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1300773&r1=1300772&r2=1300773&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Wed Mar 14 22:53:48 2012
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-675 checkMultipleTimeInstances doesn't work for EL extensions. EX: ${coord:formatTime(coord:current(0),'yyyy-MM-dd')} (sriksun via tucu)
OOZIE-749 oozie tests doesn't delete the files in tmp directories (virag via tucu)
OOZIE-761 Fixes in CoordELFunctions and testcases for Hadoop 0.23 (tucu)
OOZIE-760 oozied.sh needs to be able to pick up CATALINA_HOME (rvs via tucu)