You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by mo...@apache.org on 2013/05/23 23:27:12 UTC
svn commit: r1485857 - in /oozie/trunk: ./ bin/
core/src/main/java/org/apache/oozie/command/coord/
core/src/main/java/org/apache/oozie/coord/
core/src/test/java/org/apache/oozie/command/coord/
core/src/test/java/org/apache/oozie/test/ core/src/test/res...
Author: mona
Date: Thu May 23 21:27:11 2013
New Revision: 1485857
URL: http://svn.apache.org/r1485857
Log:
OOZIE-674 resolveInstanceRange doesn't work for EL extensions (shwethags via mona)
Added:
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/CoordELExtensions.java
oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordELExtensions.java
oozie/trunk/core/src/test/resources/coord-job-for-elext.xml
oozie/trunk/core/src/test/resources/oozie-site-coordel.xml
Modified:
oozie/trunk/bin/test-patch-05-patch-raw-analysis
oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
oozie/trunk/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
oozie/trunk/release-log.txt
Modified: oozie/trunk/bin/test-patch-05-patch-raw-analysis
URL: http://svn.apache.org/viewvc/oozie/trunk/bin/test-patch-05-patch-raw-analysis?rev=1485857&r1=1485856&r2=1485857&view=diff
==============================================================================
--- oozie/trunk/bin/test-patch-05-patch-raw-analysis (original)
+++ oozie/trunk/bin/test-patch-05-patch-raw-analysis Thu May 23 21:27:11 2013
@@ -30,7 +30,8 @@ cleanupAndExit() {
}
###############################################################################
printUsage() {
- echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR> --reportdir=<REPORT DIR> --summaryfile=<SUMMARY FILE>)"
+ echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR> --reportdir=<REPORT DIR> --summaryfile=<SUMMARY FILE> "
+ echo "--patchfile=<PATCH FILE>)"
echo
}
###############################################################################
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java?rev=1485857&r1=1485856&r2=1485857&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java Thu May 23 21:27:11 2013
@@ -63,28 +63,36 @@ public class CoordCommandUtils {
* <p/>
*
* @param function
- * @param event
- * @param appInst
- * @param conf
* @param restArg
* @return int instanceNumber
* @throws Exception
*/
- public static int getInstanceNumber(String function, Element event, SyncCoordAction appInst, Configuration conf,
- StringBuilder restArg) throws Exception {
- ELEvaluator eval = CoordELEvaluator
- .createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf);
- String newFunc = CoordELFunctions.evalAndWrap(eval, function);
- int funcType = getFuncType(newFunc);
+ public static int getInstanceNumber(String function, StringBuilder restArg) throws Exception {
+ int funcType = getFuncType(function);
if (funcType == CURRENT || funcType == LATEST) {
- return parseOneArg(newFunc);
+ return parseOneArg(function);
}
else {
- return parseMoreArgs(newFunc, restArg);
+ return parseMoreArgs(function, restArg);
}
}
- private static int parseOneArg(String funcName) throws Exception {
+ /**
+ * Evaluates function for coord-action-create-inst tag
+ * @param event
+ * @param appInst
+ * @param conf
+ * @param function
+ * @return evaluation result
+ * @throws Exception
+ */
+ private static String evaluateInstanceFunction(Element event, SyncCoordAction appInst, Configuration conf,
+ String function) throws Exception {
+ ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf);
+ return CoordELFunctions.evalAndWrap(eval, function);
+ }
+
+ public static int parseOneArg(String funcName) throws Exception {
int firstPos = funcName.indexOf("(");
int lastPos = funcName.lastIndexOf(")");
if (firstPos >= 0 && lastPos > firstPos) {
@@ -181,17 +189,17 @@ public class CoordCommandUtils {
Element eStartInst = event.getChild("start-instance", event.getNamespace());
Element eEndInst = event.getChild("end-instance", event.getNamespace());
if (eStartInst != null && eEndInst != null) {
- String strStart = eStartInst.getTextTrim();
- String strEnd = eEndInst.getTextTrim();
+ String strStart = evaluateInstanceFunction(event, appInst, conf, eStartInst.getTextTrim());
+ String strEnd = evaluateInstanceFunction(event, appInst, conf, eEndInst.getTextTrim());
checkIfBothSameType(strStart, strEnd);
StringBuilder restArg = new StringBuilder(); // To store rest
// arguments for
// future
// function
- int startIndex = getInstanceNumber(strStart, event, appInst, conf, restArg);
+ int startIndex = getInstanceNumber(strStart, restArg);
String startRestArg = restArg.toString();
restArg.delete(0, restArg.length());
- int endIndex = getInstanceNumber(strEnd, event, appInst, conf, restArg);
+ int endIndex = getInstanceNumber(strEnd, restArg);
String endRestArg = restArg.toString();
int funcType = getFuncType(strStart);
if (funcType == OFFSET) {
Modified: oozie/trunk/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java?rev=1485857&r1=1485856&r2=1485857&view=diff
==============================================================================
--- oozie/trunk/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java (original)
+++ oozie/trunk/core/src/main/java/org/apache/oozie/coord/CoordELFunctions.java Thu May 23 21:27:11 2013
@@ -1250,7 +1250,7 @@ public class CoordELFunctions {
* @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
* the dataset.
*/
- private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) {
+ public static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) {
ELEvaluator eval = ELEvaluator.getCurrent();
return getCurrentInstance(effectiveTime, instanceCount, eval);
}
@@ -1294,7 +1294,7 @@ public class CoordELFunctions {
return current;
}
- private static Calendar getEffectiveNominalTime() {
+ public static Calendar getEffectiveNominalTime() {
Date datasetInitialInstance = getInitialInstance();
TimeZone dsTZ = getDatasetTZ();
// Convert Date to Calendar for corresponding TZ
@@ -1356,7 +1356,7 @@ public class CoordELFunctions {
/**
* @return dataset TimeZone
*/
- private static TimeZone getDatasetTZ() {
+ public static TimeZone getDatasetTZ() {
ELEvaluator eval = ELEvaluator.getCurrent();
return getDatasetTZ(eval);
}
Added: oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/CoordELExtensions.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/CoordELExtensions.java?rev=1485857&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/CoordELExtensions.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/CoordELExtensions.java Thu May 23 21:27:11 2013
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.oozie.command.coord;
+
+import java.util.Calendar;
+
+import org.apache.oozie.coord.CoordELFunctions;
+import org.apache.oozie.util.ELEvaluator;
+
+public class CoordELExtensions {
+ private static final String PREFIX = "coordext:";
+
+ public static String ph1_today_echo(int hr, int min) {
+ ELEvaluator eval = ELEvaluator.getCurrent();
+ eval.setVariable(".wrap", "true");
+ return PREFIX + "today(" + hr + ", " + min + ")"; // Unresolved
+ }
+
+ public static String ph2_today_inst(int hr, int min) throws Exception {
+ Calendar nominalInstanceCal = CoordELFunctions.getEffectiveNominalTime();
+ if (nominalInstanceCal == null) {
+ return "";
+ }
+
+ Calendar dsInstanceCal = Calendar.getInstance(CoordELFunctions.getDatasetTZ());
+ dsInstanceCal.setTime(nominalInstanceCal.getTime());
+ dsInstanceCal.set(Calendar.HOUR_OF_DAY, hr);
+ dsInstanceCal.set(Calendar.MINUTE, min);
+ dsInstanceCal.set(Calendar.SECOND, 0);
+ dsInstanceCal.set(Calendar.MILLISECOND, 0);
+
+ int[] instCnt = new int[1];
+ Calendar compInstCal = CoordELFunctions
+ .getCurrentInstance(dsInstanceCal.getTime(), instCnt);
+ if (compInstCal == null) {
+ return "";
+ }
+ int dsInstanceCnt = instCnt[0];
+
+ compInstCal = CoordELFunctions.getCurrentInstance(nominalInstanceCal.getTime(), instCnt);
+ if (compInstCal == null) {
+ return "";
+ }
+ int nominalInstanceCnt = instCnt[0];
+
+ return "coord:current(" + (dsInstanceCnt - nominalInstanceCnt) + ")";
+ }
+
+ public static String ph2_today(int hr, int min) throws Exception {
+ String inst = ph2_today_inst(hr, min);
+ return evaluateCurrent(inst);
+ }
+
+ private static String evaluateCurrent(String curExpr) throws Exception {
+ if (curExpr.equals("")) {
+ return curExpr;
+ }
+
+ int inst = CoordCommandUtils.parseOneArg(curExpr);
+ return CoordELFunctions.ph2_coord_current(inst);
+ }
+}
\ No newline at end of file
Added: oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordELExtensions.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordELExtensions.java?rev=1485857&view=auto
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordELExtensions.java (added)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/command/coord/TestCoordELExtensions.java Thu May 23 21:27:11 2013
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.oozie.command.coord;
+
+import java.io.File;
+import java.util.Date;
+import java.util.HashMap;
+
+import org.apache.oozie.CoordinatorActionBean;
+import org.apache.oozie.CoordinatorJobBean;
+import org.apache.oozie.client.CoordinatorJob;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.store.CoordinatorStore;
+import org.apache.oozie.store.StoreException;
+import org.apache.oozie.test.XDataTestCase;
+import org.apache.oozie.util.DateUtils;
+
+public class TestCoordELExtensions extends XDataTestCase {
+ private Services services;
+ private String[] excludedServices = { "org.apache.oozie.service.StatusTransitService",
+ "org.apache.oozie.service.PauseTransitService",
+ "org.apache.oozie.service.RecoveryService" };
+
+ @Override
+ protected void setUp() throws Exception {
+ sysProps = new HashMap<String, String>();
+ setSystemProperty("oozie.test.config.file", new File(OOZIE_SRC_DIR,
+ "core/src/test/resources/oozie-site-coordel.xml").getAbsolutePath());
+ super.setUp();
+ services = new Services();
+ setClassesToBeExcluded(services.getConf(), excludedServices);
+ services.init();
+ cleanUpDBTables();
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ services.destroy();
+ super.tearDown();
+ }
+
+ public void testCoordELActionMater() throws Exception {
+ Date startTime = DateUtils.parseDateUTC("2009-03-06T010:00Z");
+ Date endTime = DateUtils.parseDateUTC("2009-03-07T12:00Z");
+ CoordinatorJobBean job = createCoordJob("coord-job-for-elext.xml",
+ CoordinatorJob.Status.PREMATER, startTime, endTime, false, false, 0);
+ addRecordToCoordJobTable(job);
+
+ new CoordActionMaterializeCommand(job.getId(), startTime, endTime).call();
+ checkCoordAction(job.getId() + "@1");
+ }
+
+ protected CoordinatorActionBean checkCoordAction(String actionId) throws StoreException {
+ CoordinatorStore store = new CoordinatorStore(false);
+ try {
+ CoordinatorActionBean action = store.getCoordinatorAction(actionId, false);
+ assertEquals(
+ "file://#testDir/2009/03/06/00/_SUCCESS#file://#testDir/2009/03/05/23/_SUCCESS",
+ action.getMissingDependencies());
+ return action;
+ } catch (StoreException se) {
+ se.printStackTrace();
+ fail("Action ID " + actionId + " was not stored properly in db");
+ }
+ return null;
+ }
+}
\ No newline at end of file
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java?rev=1485857&r1=1485856&r2=1485857&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java Thu May 23 21:27:11 2013
@@ -121,6 +121,25 @@ public abstract class XDataTestCase exte
}
/**
+ * Inserts the passed coord job
+ * @param coord job bean
+ * @throws Exception
+ */
+ protected void addRecordToCoordJobTable(CoordinatorJobBean coordJob) throws Exception {
+ try {
+ JPAService jpaService = Services.get().get(JPAService.class);
+ assertNotNull(jpaService);
+ CoordJobInsertJPAExecutor coordInsertCmd = new CoordJobInsertJPAExecutor(coordJob);
+ jpaService.execute(coordInsertCmd);
+ }
+ catch (JPAExecutorException je) {
+ je.printStackTrace();
+ fail("Unable to insert the test coord job record to table");
+ throw je;
+ }
+ }
+
+ /**
* Insert coord job for testing.
*
* @param status coord job status
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1485857&r1=1485856&r2=1485857&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java Thu May 23 21:27:11 2013
@@ -92,13 +92,13 @@ import org.apache.oozie.util.XLog;
* From within testcases, system properties must be changed using the {@link #setSystemProperty} method.
*/
public abstract class XTestCase extends TestCase {
- private Map<String, String> sysProps;
+ protected Map<String, String> sysProps;
private String testCaseDir;
private String testCaseConfDir;
private String hadoopVersion;
protected XLog log = new XLog(LogFactory.getLog(getClass()));
- private static File OOZIE_SRC_DIR = null;
+ protected static File OOZIE_SRC_DIR = null;
private static final String OOZIE_TEST_PROPERTIES = "oozie.test.properties";
public static float WAITFOR_RATIO = Float.parseFloat(System.getProperty("oozie.test.waitfor.ratio", "1"));
Added: oozie/trunk/core/src/test/resources/coord-job-for-elext.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/resources/coord-job-for-elext.xml?rev=1485857&view=auto
==============================================================================
--- oozie/trunk/core/src/test/resources/coord-job-for-elext.xml (added)
+++ oozie/trunk/core/src/test/resources/coord-job-for-elext.xml Thu May 23 21:27:11 2013
@@ -0,0 +1,63 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<coordinator-app xmlns='uri:oozie:coordinator:0.2' name='NAME'
+ frequency="1" start='2009-02-01T01:00Z' end='2009-02-03T23:59Z'
+ timezone='UTC' freq_timeunit='DAY' end_of_duration='NONE'>
+ <controls>
+ <timeout>10</timeout>
+ <concurrency>2</concurrency>
+ <execution>LIFO</execution>
+ </controls>
+ <input-events>
+ <data-in name='A' dataset='a'>
+ <dataset name='a' frequency='1' initial-instance='2009-02-01T00:00Z'
+ timezone='UTC' freq_timeunit='HOUR' end_of_duration='NONE'>
+ <uri-template>file://#testDir/${YEAR}/${MONTH}/${DAY}/${HOUR}
+ </uri-template>
+ </dataset>
+ <start-instance>${coordext:today(-1,0)}</start-instance>
+ <end-instance>${coordext:today(0,0)}</end-instance>
+ </data-in>
+ </input-events>
+ <output-events>
+ <data-out name='LOCAL_A' dataset='local_a'>
+ <dataset name='local_a' frequency='1'
+ initial-instance='2009-02-01T00:00Z' timezone='UTC'
+ freq_timeunit='HOUR' end_of_duration='NONE'>
+ <uri-template>file://#testDir/${YEAR}/${DAY}
+ </uri-template>
+ </dataset>
+ <instance>${coordext:today(0,0)}</instance>
+ </data-out>
+ </output-events>
+ <action>
+ <workflow>
+ <app-path>hdfs:///tmp/workflows/</app-path>
+ <configuration>
+ <property>
+ <name>inputA</name>
+ <value>${coord:dataIn('A')}</value>
+ </property>
+ <property>
+ <name>inputB</name>
+ <value>${coord:dataOut('LOCAL_A')}</value>
+ </property>
+ </configuration>
+ </workflow>
+ </action>
+</coordinator-app>
\ No newline at end of file
Added: oozie/trunk/core/src/test/resources/oozie-site-coordel.xml
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/resources/oozie-site-coordel.xml?rev=1485857&view=auto
==============================================================================
--- oozie/trunk/core/src/test/resources/oozie-site-coordel.xml (added)
+++ oozie/trunk/core/src/test/resources/oozie-site-coordel.xml Thu May 23 21:27:11 2013
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-job-submit-instances
+ </name>
+ <value>
+ coordext:today=org.apache.oozie.command.coord.CoordELExtensions#ph1_today_echo
+ </value>
+ <description>
+ EL functions declarations, separated by commas,
+ format is [PREFIX:]NAME=CLASS#METHOD.
+ This property is a
+ convenience property to add extensions to the built in
+ executors without having to
+ include all the built in ones.
+ </description>
+ </property>
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-action-create-inst
+ </name>
+ <value>
+ coordext:today=org.apache.oozie.command.coord.CoordELExtensions#ph2_today_inst
+ </value>
+ <description>
+ EL functions declarations, separated by commas,
+ format is [PREFIX:]NAME=CLASS#METHOD.
+ This property is a
+ convenience property to add extensions to the built in
+ executors without having to
+ include all the built in ones.
+ </description>
+ </property>
+ <property>
+ <name>oozie.service.ELService.ext.functions.coord-action-create
+ </name>
+ <value>
+ coordext:today=org.apache.oozie.command.coord.CoordELExtensions#ph2_today
+ </value>
+ <description>
+ EL functions declarations, separated by commas,
+ format is [PREFIX:]NAME=CLASS#METHOD.
+ This property is a
+ convenience property to add extensions to the built in
+ executors without having to
+ include all the built in ones.
+ </description>
+ </property>
+</configuration>
\ No newline at end of file
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1485857&r1=1485856&r2=1485857&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Thu May 23 21:27:11 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-674 resolveInstanceRange doesn't work for EL extensions (shwethags via mona)
OOZIE-1384 Make Uber Mode not the default (rkanter via virag)
OOZIE-1386 NPE in XOozieClient if fs.default.name is not defined but fs.defaultFS is (wypoon via rkanter)
OOZIE-1387 Proxysubmission from the Oozie client doesn't allow the mapreduce API (rkanter)