You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2015/10/27 21:45:00 UTC
oozie git commit: OOZIE-2368 coord:dateOffset and coord:dateTzOffset
can't be used for coord initial-instance
Repository: oozie
Updated Branches:
refs/heads/master 2910636ee -> f713d6bd7
OOZIE-2368 coord:dateOffset and coord:dateTzOffset can't be used for coord initial-instance
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f713d6bd
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f713d6bd
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f713d6bd
Branch: refs/heads/master
Commit: f713d6bd72ab1c49740c5989cfde1d52ac42b34b
Parents: 2910636
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Tue Oct 27 13:48:42 2015 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Tue Oct 27 13:48:42 2015 -0700
----------------------------------------------------------------------
.../command/coord/CoordSubmitXCommand.java | 5 +-
.../apache/oozie/coord/CoordELEvaluator.java | 9 ++--
core/src/main/resources/oozie-default.xml | 23 +++++++--
.../command/coord/TestCoordSubmitXCommand.java | 50 ++++++++++++++++++++
.../oozie/coord/TestCoordELEvaluator.java | 17 -------
release-log.txt | 1 +
6 files changed, 78 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
index 587b983..d4d1c08 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
@@ -143,6 +143,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
private ELEvaluator evalAction = null;
private ELEvaluator evalSla = null;
private ELEvaluator evalTimeout = null;
+ private ELEvaluator evalInitialInstance = null;
static {
String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY,
@@ -661,6 +662,8 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
evalInst = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-instances");
evalAction = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-action-start");
evalTimeout = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-wait-timeout");
+ evalInitialInstance = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-initial-instance");
+
}
/**
@@ -969,7 +972,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
.toString() : ((TimeUnit) evalFreq.getVariable("timeunit")).toString());
addAnAttribute("end_of_duration", dsElem, evalFreq.getVariable("endOfDuration") == null ? TimeUnit.NONE
.toString() : ((TimeUnit) evalFreq.getVariable("endOfDuration")).toString());
- val = resolveAttribute("initial-instance", dsElem, evalNofuncs);
+ val = resolveAttribute("initial-instance", dsElem, evalInitialInstance);
ParamChecker.checkDateOozieTZ(val, "initial-instance");
checkInitialInstance(val);
val = resolveAttribute("timezone", dsElem, evalNofuncs);
http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java b/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java
index d495dfa..8b2f456 100644
--- a/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java
+++ b/core/src/main/java/org/apache/oozie/coord/CoordELEvaluator.java
@@ -73,7 +73,7 @@ public class CoordELEvaluator {
Configuration conf) throws Exception {
ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
setConfigToEval(eval, conf);
- SyncCoordDataset ds = getDSObject(eval, event);
+ SyncCoordDataset ds = getDSObject(event);
CoordELFunctions.configureEvaluator(eval, ds, appInst);
return eval;
}
@@ -104,7 +104,7 @@ public class CoordELEvaluator {
throws Exception {
ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
setConfigToEval(eval, conf);
- SyncCoordDataset ds = getDSObject(eval, dEvent);
+ SyncCoordDataset ds = getDSObject(dEvent);
SyncCoordAction appInst = new SyncCoordAction();
appInst.setNominalTime(nominalTime);
appInst.setActualTime(actualTime);
@@ -254,12 +254,11 @@ public class CoordELEvaluator {
* @return
* @throws Exception
*/
- private static SyncCoordDataset getDSObject(ELEvaluator eval, Element eData) throws Exception {
+ private static SyncCoordDataset getDSObject(Element eData) throws Exception {
SyncCoordDataset ds = new SyncCoordDataset();
Element eDataset = eData.getChild("dataset", eData.getNamespace());
// System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
- String expr = eDataset.getAttributeValue("initial-instance");
- Date initInstance = DateUtils.parseDateOozieTZ(eval.evaluate(expr, String.class));
+ Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
ds.setInitInstance(initInstance);
if (eDataset.getAttributeValue("frequency") != null) {
int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 649691d..faf3740 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -583,7 +583,7 @@
<!-- List of supported groups for ELService -->
<property>
<name>oozie.service.ELService.groups</name>
- <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout,bundle-submit</value>
+ <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout,bundle-submit,coord-job-submit-initial-instance</value>
<description>List of groups for different ELServices</description>
</property>
@@ -798,6 +798,18 @@
</property>
<property>
+ <name>oozie.service.ELService.functions.coord-job-submit-initial-instance</name>
+ <value>
+ ${oozie.service.ELService.functions.coord-job-submit-nofuncs},
+ coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset
+ </value>
+ <description>
+ EL functions for coord job submit initial instance, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.ELService.ext.functions.coord-job-submit-freq</name>
<value>
</value>
@@ -932,7 +944,9 @@
coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf,
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph1_coord_absolute_echo,
- hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf
+ hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf,
+ coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateTzOffset_echo
</value>
<description>
EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
@@ -1104,7 +1118,6 @@
coord:current=org.apache.oozie.coord.CoordELFunctions#ph2_coord_current,
coord:currentRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_currentRange,
coord:offset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_offset,
- coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset,
coord:latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo,
coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latestRange_echo,
coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo,
@@ -1173,7 +1186,9 @@
coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user,
coord:absolute=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_echo,
coord:absoluteRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_absolute_range,
- hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf
+ hadoop:conf=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_conf,
+ coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateOffset,
+ coord:dateTzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_dateTzOffset
</value>
<description>
EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
index e17d1b5..52eb9dd 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
@@ -28,12 +28,14 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.BundleJobBean;
+import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.client.Job;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.rest.RestConstants;
import org.apache.oozie.command.CommandException;
+import org.apache.oozie.executor.jpa.CoordActionQueryExecutor;
import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor;
import org.apache.oozie.executor.jpa.CoordJobQueryExecutor;
import org.apache.oozie.executor.jpa.JPAExecutorException;
@@ -1519,4 +1521,52 @@ public class TestCoordSubmitXCommand extends XDataTestCase {
Element eSla = eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"));
assertEquals(SLAOperations.getTagElement(eSla, "max-duration"), "${5 * MINUTES}");
}
+
+ public void testSubmitDateOffset() throws Exception {
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ String appXml = "<coordinator-app name=\"${NAME}\" frequency=\"${coord:days(1)}\" start=\"${startDate}\" "
+ + "end=\"${endDate}\" timezone=\"UTC\" "
+ + "xmlns=\"uri:oozie:coordinator:0.3\"> <controls> <timeout>10</timeout> <concurrency>2</concurrency> "
+ + "<execution>LIFO</execution> </controls> <datasets> "
+ + "<dataset name=\"a\" frequency=\"${coord:days(1)}\" "
+ + "initial-instance=\"${coord:dateOffset(startDate,-2,'DAY')}\" "
+ + "timezone=\"UTC\"> <uri-template>file:///tmp/coord/a/${YEAR}/${DAY}"
+ + "</uri-template> </dataset> "
+ + "<dataset name=\"b\" frequency=\"${coord:days(1)}\" initial-instance=\"${coord:dateTzOffset(startDate,'UTC')}\" "
+ + "timezone=\"UTC\"> <uri-template>file:///tmp/coord/b/${YEAR}/${DAY}</uri-template> "
+ + "<done-flag>${MY_DONE_FLAG}</done-flag> </dataset>"
+ + "</datasets> <input-events> "
+ + "<data-in name=\"A\" dataset=\"a\"> <start-instance>${coord:absolute(coord:dateOffset(startDate,-1,'DAY'))}"
+ + "</start-instance><end-instance>${coord:current(0)}</end-instance></data-in> "
+ + "<data-in name=\"B\" dataset=\"b\"> <start-instance>${coord:absolute(coord:dateTzOffset(startDate,'UTC'))}"
+ + "</start-instance><end-instance>${coord:current(0)}</end-instance></data-in> "
+ + "</input-events> "
+ + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> "
+ + "</workflow> </action> </coordinator-app>";
+ writeToFile(appXml, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("MY_DONE_FLAG", "complete");
+ conf.set("NAME", "test_app_name");
+ conf.set("startDate", "2009-02-03T00:00Z");
+ conf.set("endDate", "2009-03-03T00:00Z");
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ String jobId = sc.call();
+ CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get(
+ CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, jobId);
+ assertTrue(job.getJobXml()
+ .contains("dataset name=\"a\" frequency=\"1\" initial-instance=\"2009-02-01T00:00Z\""));
+ assertTrue(job.getJobXml()
+ .contains("dataset name=\"b\" frequency=\"1\" initial-instance=\"2009-02-03T00:00Z\""));
+ new CoordMaterializeTransitionXCommand(jobId, 3600).call();
+ CoordinatorActionBean action = CoordActionQueryExecutor.getInstance().get(
+ CoordActionQueryExecutor.CoordActionQuery.GET_COORD_ACTION, jobId+"@1");
+ action.getActionXml();
+ assertTrue(action.getActionXml()
+ .contains("tmp/coord/a/2009/02"));
+ assertTrue(action.getActionXml()
+ .contains("tmp/coord/b/2009/03"));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java b/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java
index b67e77e..a429665 100644
--- a/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java
+++ b/core/src/test/java/org/apache/oozie/coord/TestCoordELEvaluator.java
@@ -171,23 +171,6 @@ public class TestCoordELEvaluator extends XTestCase {
assertEquals("2009-09-08T00:00Z", eval.evaluate(expr, String.class));
}
- public void testCreateInstancesELEvaluatorWithDateOffset() throws Exception {
- SyncCoordAction appInst = new SyncCoordAction();
- appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-09-08T01:00Z"));
- appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
- appInst.setTimeUnit(TimeUnit.MINUTE);
- String dataEvntXML = "<data-in name=\"A\" dataset=\"a\"><uris>file:///tmp/coord/US/2009/1/30|file:///tmp/coord/"
- + "US/2009/1/31</uris>";
- dataEvntXML += "<dataset name=\"a\" frequency=\"1440\" initial-instance=\"${coord:dateOffset('2009-09-08T23:59Z'"
- + ", 2, 'MINUTE')}\" freq_timeunit=\"MINUTE\" timezone=\"UTC\" end_of_duration=\"NONE\">";
- dataEvntXML += "<uri-template>file:///tmp/coord/US/${YEAR}/${MONTH}/${DAY}</uri-template></dataset></data-in>";
- Element event = XmlUtils.parseXml(dataEvntXML);
- Configuration conf = new XConfiguration(new StringReader(getConfString()));
- ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
- SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(CoordELFunctions.DATASET);
- assertEquals("2009-09-09T00:01Z", DateUtils.formatDateOozieTZ(ds.getInitInstance()));
- }
-
public void testCreateLazyEvaluator() throws Exception {
// Configuration conf = new
// XConfiguration(IOUtils.getResourceAsReader("org/apache/oozie/coord/conf.xml",
http://git-wip-us.apache.org/repos/asf/oozie/blob/f713d6bd/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 7f687c6..9a0b9c5 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2368 coord:dateOffset and coord:dateTzOffset can't be used for coord initial-instance (puru)
OOZIE-2369 coord:offset doesn't resolve correctly (puru)
OOZIE-2251 Expose instrumental matrices in Realtime Graphing tool (nperiwal via rkanter)
OOZIE-2314 Unable to kill old instance child job by workflow or coord rerun by Launcher (jaydeepvishwakarma via rkanter)