You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ry...@apache.org on 2014/02/21 02:10:44 UTC
git commit: OOZIE-1629 EL function in is not evaluated
properly (ryota)
Repository: oozie
Updated Branches:
refs/heads/master c729cab62 -> 5508ec7a6
OOZIE-1629 EL function in <timeout> is not evaluated properly (ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5508ec7a
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5508ec7a
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5508ec7a
Branch: refs/heads/master
Commit: 5508ec7a660b00e0ea0a133d60ff49444c70ce1f
Parents: c729cab
Author: egashira <ry...@yahoo.com>
Authored: Thu Feb 20 16:15:09 2014 -0800
Committer: egashira <ry...@yahoo.com>
Committed: Thu Feb 20 16:15:09 2014 -0800
----------------------------------------------------------------------
.../command/coord/CoordSubmitXCommand.java | 24 ++++++-
core/src/main/resources/oozie-default.xml | 15 ++++-
.../command/coord/TestCoordSubmitXCommand.java | 68 ++++++++++++++++++++
.../site/twiki/CoordinatorFunctionalSpec.twiki | 2 +
release-log.txt | 1 +
5 files changed, 107 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
index 609b20c..712fe51 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
@@ -140,6 +140,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
private ELEvaluator evalInst = null;
private ELEvaluator evalAction = null;
private ELEvaluator evalSla = null;
+ private ELEvaluator evalTimeout = null;
static {
String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY,
@@ -625,6 +626,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
evalInst = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-submit-instances");
evalSla = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-sla-submit");
evalAction = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-action-start");
+ evalTimeout = CoordELEvaluator.createELEvaluatorForGroup(conf, "coord-job-wait-timeout");
}
/**
@@ -683,8 +685,26 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
coordJob.setTimeZone(val);
// controls
- val = resolveTagContents("timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalFreq);
- if (val == "") {
+ val = resolveTagContents("timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalTimeout);
+ if (val != null && val != "") {
+ int t = Integer.parseInt(val);
+ tmp = (evalTimeout.getVariable("timeunit") == null) ? TimeUnit.MINUTE : ((TimeUnit) evalTimeout
+ .getVariable("timeunit"));
+ switch (tmp) {
+ case HOUR:
+ val = String.valueOf(t * 60);
+ break;
+ case DAY:
+ val = String.valueOf(t * 60 * 24);
+ break;
+ case MONTH:
+ val = String.valueOf(t * 60 * 24 * 30);
+ break;
+ default:
+ break;
+ }
+ }
+ else {
val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 26e3f09..0fd1796 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -501,7 +501,7 @@
<!-- List of supported groups for ELService -->
<property>
<name>oozie.service.ELService.groups</name>
- <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start</value>
+ <value>job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start,coord-job-wait-timeout</value>
<description>List of groups for different ELServices</description>
</property>
@@ -696,6 +696,19 @@
</property>
<property>
+ <name>oozie.service.ELService.functions.coord-job-wait-timeout</name>
+ <value>
+ coord:days=org.apache.oozie.coord.CoordELFunctions#ph1_coord_days,
+ coord:months=org.apache.oozie.coord.CoordELFunctions#ph1_coord_months,
+ coord:hours=org.apache.oozie.coord.CoordELFunctions#ph1_coord_hours,
+ coord:minutes=org.apache.oozie.coord.CoordELFunctions#ph1_coord_minutes
+ </value>
+ <description>
+ EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.ELService.ext.functions.coord-job-submit-freq</name>
<value>
</value>
http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
index dc289b4..88756f6 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
@@ -1151,4 +1151,72 @@ public class TestCoordSubmitXCommand extends XDataTestCase {
}
}
}
+
+ /**
+ * Test timeout setting
+ *
+ * @throws Exception
+ */
+ public void testSubmitWithTimeout() throws Exception {
+ Configuration conf = new XConfiguration();
+ File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
+ // timeout unit = DAY
+ String appXml1 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" "
+ + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" "
+ + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> "
+ + "<controls> <timeout>${coord:days(10)}</timeout> </controls> "
+ + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> "
+ + "</action> </coordinator-app>";
+ writeToFile(appXml1, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
+ String jobId = sc.call();
+ assertEquals(jobId.substring(jobId.length() - 2), "-C");
+ CoordinatorJobBean job = checkCoordJobs(jobId);
+ assertEquals(job.getTimeout(), 14400);
+ // timeout unit = HOUR
+ String appXml2 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" "
+ + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" "
+ + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> "
+ + "<controls> <timeout>${coord:hours(10)}</timeout> </controls> "
+ + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> "
+ + "</action> </coordinator-app>";
+ writeToFile(appXml2, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ sc = new CoordSubmitXCommand(conf);
+ jobId = sc.call();
+ job = checkCoordJobs(jobId);
+ assertEquals(job.getTimeout(), 600);
+ // timeout unit = MINUTE
+ String appXml3 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" "
+ + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" "
+ + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> "
+ + "<controls> <timeout>${coord:minutes(10)}</timeout> </controls> "
+ + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> "
+ + "</action> </coordinator-app>";
+ writeToFile(appXml3, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ sc = new CoordSubmitXCommand(conf);
+ jobId = sc.call();
+ job = checkCoordJobs(jobId);
+ assertEquals(job.getTimeout(), 10);
+ // timeout unit = MONTH
+ String appXml4 = "<coordinator-app name=\"NAME\" frequency=\"${coord:months(1)}\" "
+ + "start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\" "
+ + "xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2'> "
+ + "<controls> <timeout>${coord:months(1)}</timeout> </controls> "
+ + "<action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> "
+ + "</action> </coordinator-app>";
+ writeToFile(appXml4, appPathFile);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ sc = new CoordSubmitXCommand(conf);
+ jobId = sc.call();
+ job = checkCoordJobs(jobId);
+ assertEquals(job.getTimeout(), 43200);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
index 7cfbc0c..44a3d54 100644
--- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
@@ -208,6 +208,8 @@ Coordinator Frequencies can also be expressed using cron syntax.
| =${coord:months(int n)}= | _variable_ | =${coord:months(1)}= --> minutes in a 1 full month from the current date |
| =${cron syntax}= | _variable_ | =${0,10 15 * * 2-6}= --> a job that runs every weekday at 3:00pm and 3:10pm UTC time|
+Note that, though =${coord:days(int n)}= and =${coord:months(int n)}= EL functions are used to calculate minutes precisely including variations due to daylight saving time for Frequency representation, when specified for coordinator timeout interval, one day is calculated as 24 hours and one month is calculated as 30 days for simplicity.
+
---++++ 4.4.1. The coord:days(int n) and coord:endOfDays(int n) EL functions
The =${coord:days(int n)}= and =${coord:endOfDays(int n)}= EL functions should be used to handle day based frequencies.
http://git-wip-us.apache.org/repos/asf/oozie/blob/5508ec7a/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 2248b40..40917d2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1629 EL function in <timeout> is not evaluated properly (ryota)
OOZIE-1618 dryrun should check variable substitution in workflow.xml (bowenzhangusa via rkanter)
OOZIE-1681 Sqoop sharelib has no hsqldb jar version (Ostap via rkanter)
OOZIE-1691 StackOverflowError in TimestampedMessageParser.parseNextLine() (puru via rkanter)