You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/08/14 01:21:17 UTC
svn commit: r1372663 [3/3] - in /incubator/oozie/trunk: ./
core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/command/bundle/
core/src/main/java/org/apache/oozie/command/coord/
core/src/main/java/org/apache/oozie/coord/ core/src/m...
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XDataTestCase.java Mon Aug 13 23:21:15 2012
@@ -256,8 +256,8 @@ public abstract class XDataTestCase exte
coordJob.setConcurrency(1);
coordJob.setMatThrottling(1);
try {
- coordJob.setStartTime(DateUtils.parseDateUTC("2009-12-15T01:00Z"));
- coordJob.setEndTime(DateUtils.parseDateUTC("2009-12-17T01:00Z"));
+ coordJob.setStartTime(DateUtils.parseDateOozieTZ("2009-12-15T01:00Z"));
+ coordJob.setEndTime(DateUtils.parseDateOozieTZ("2009-12-17T01:00Z"));
}
catch (Exception e) {
e.printStackTrace();
@@ -496,6 +496,10 @@ public abstract class XDataTestCase exte
return action;
}
+ protected CoordinatorActionBean createCoordAction(String jobId, int actionNum, CoordinatorAction.Status status,
+ String resourceXmlName, int pending) throws Exception {
+ return createCoordAction(jobId, actionNum, status, resourceXmlName, pending, "Z");
+ }
/**
* Create coord action bean
*
@@ -508,11 +512,12 @@ public abstract class XDataTestCase exte
* @throws Exception thrown if unable to create coord action bean
*/
protected CoordinatorActionBean createCoordAction(String jobId, int actionNum, CoordinatorAction.Status status,
- String resourceXmlName, int pending) throws Exception {
+ String resourceXmlName, int pending, String oozieTimeZoneMask) throws Exception {
String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, actionNum + "");
Path appPath = new Path(getFsTestCaseDir(), "coord");
String actionXml = getCoordActionXml(appPath, resourceXmlName);
- String actionNomialTime = getActionNominalTime(actionXml);
+ actionXml = actionXml.replace("${TZ}", oozieTimeZoneMask);
+ String actionNominalTime = getActionNominalTime(actionXml);
CoordinatorActionBean action = new CoordinatorActionBean();
action.setId(actionId);
@@ -521,7 +526,7 @@ public abstract class XDataTestCase exte
action.setActionNumber(actionNum);
action.setPending(pending);
try {
- action.setNominalTime(DateUtils.parseDateUTC(actionNomialTime));
+ action.setNominalTime(DateUtils.parseDateOozieTZ(actionNominalTime));
}
catch (Exception e) {
e.printStackTrace();
@@ -786,8 +791,8 @@ public abstract class XDataTestCase exte
protected String getCoordJobXml(Path appPath, Date start, Date end) {
String startDateStr = null, endDateStr = null;
try {
- startDateStr = DateUtils.formatDateUTC(start);
- endDateStr = DateUtils.formatDateUTC(end);
+ startDateStr = DateUtils.formatDateOozieTZ(start);
+ endDateStr = DateUtils.formatDateOozieTZ(end);
}
catch (Exception ex) {
ex.printStackTrace();
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestCoordActionsInDateRange.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestCoordActionsInDateRange.java?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestCoordActionsInDateRange.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestCoordActionsInDateRange.java Mon Aug 13 23:21:15 2012
@@ -26,11 +26,9 @@ import org.apache.oozie.ErrorCode;
import org.apache.oozie.XException;
import org.apache.oozie.client.CoordinatorAction;
import org.apache.oozie.client.CoordinatorJob;
-import org.apache.oozie.command.CommandException;
import org.apache.oozie.local.LocalOozie;
import org.apache.oozie.service.Services;
import org.apache.oozie.test.XDataTestCase;
-import org.apache.oozie.util.DateUtils;
public class TestCoordActionsInDateRange extends XDataTestCase {
@@ -68,8 +66,9 @@ public class TestCoordActionsInDateRange
long nominalTimeMilliseconds = nominalTime.getTime();
long noOfMillisecondsinOneHour = 3600000;
- String date1 = DateUtils.formatDateUTC(new Date(nominalTimeMilliseconds - (noOfMillisecondsinOneHour / 2)));
- String date2 = DateUtils.formatDateUTC(new Date(nominalTimeMilliseconds + noOfMillisecondsinOneHour));
+ String date1 = DateUtils.formatDateOozieTZ(
+ new Date(nominalTimeMilliseconds - (noOfMillisecondsinOneHour / 2)));
+ String date2 = DateUtils.formatDateOozieTZ(new Date(nominalTimeMilliseconds + noOfMillisecondsinOneHour));
// Test a bad date format.
try {
@@ -123,7 +122,7 @@ public class TestCoordActionsInDateRange
assertEquals(1, noOfActions);
// Testing for the number of coordinator actions in a date range that spans from half an hour after the nominal time to 1 hour after the nominal time
- date1 = DateUtils.formatDateUTC(new Date(nominalTimeMilliseconds + (noOfMillisecondsinOneHour / 2)));
+ date1 = DateUtils.formatDateOozieTZ(new Date(nominalTimeMilliseconds + (noOfMillisecondsinOneHour / 2)));
noOfActions = CoordActionsInDateRange.getCoordActionIdsFromDates(job.getId().toString(), date1 + "::" + date2).size();
assertEquals(0, noOfActions);
}
Added: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestDateUtils.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestDateUtils.java?rev=1372663&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestDateUtils.java (added)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestDateUtils.java Mon Aug 13 23:21:15 2012
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.text.ParseException;
+import java.util.Date;
+
+public class TestDateUtils {
+
+ @After
+ public void resetToUTC() {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, DateUtils.OOZIE_PROCESSING_TIMEZONE_DEFAULT);
+ DateUtils.setConf(conf);
+ }
+
+ @Test
+ public void testOozieInUTC() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, DateUtils.OOZIE_PROCESSING_TIMEZONE_DEFAULT);
+ DateUtils.setConf(conf);
+ String s1 = "2012-08-08T12:42Z";
+ Date d = DateUtils.parseDateOozieTZ(s1);
+ Assert.assertNotNull(d);
+ String s2 = DateUtils.formatDateOozieTZ(d);
+ Assert.assertEquals(s1, s2);
+ }
+
+ @Test
+ public void testOozieInOtherTZ() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, "GMT-0300");
+ DateUtils.setConf(conf);
+ String s1 = "2012-08-08T12:42-0300";
+ Date d = DateUtils.parseDateOozieTZ(s1);
+ Assert.assertNotNull(d);
+ String s2 = DateUtils.formatDateOozieTZ(d);
+ Assert.assertEquals(s1, s2);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testOozieInOtherTZIncorrectOffset() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, "GMT-0300");
+ DateUtils.setConf(conf);
+ String s1 = "2012-08-08T12:42-0400";
+ DateUtils.parseDateOozieTZ(s1);
+ }
+
+ @Test(expected = ParseException.class)
+ public void testOozieInOtherTZInvalidOffset() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, "GMT-0300");
+ DateUtils.setConf(conf);
+ String s1 = "2012-08-08T12:42-0300x";
+ DateUtils.parseDateOozieTZ(s1);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testInvalidOozieTimeZone() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, "US/Los Angeles");
+ DateUtils.setConf(conf);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testInvalidOozieTimeZoneGMTPrefix() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, "xGMT-0300");
+ DateUtils.setConf(conf);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testInvalidOozieTimeZoneGMTPostfix() throws Exception {
+ Configuration conf = new XConfiguration();
+ conf.set(DateUtils.OOZIE_PROCESSING_TIMEZONE_KEY, "GMT-0300x");
+ DateUtils.setConf(conf);
+ }
+
+}
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParamChecker.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParamChecker.java?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParamChecker.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/util/TestParamChecker.java Mon Aug 13 23:21:15 2012
@@ -170,15 +170,15 @@ public class TestParamChecker extends XT
}
public void testCheckUTC() {
- ParamChecker.checkUTC("2009-02-01T01:00Z", "test");
+ ParamChecker.checkDateOozieTZ("2009-02-01T01:00Z", "test");
try {
- ParamChecker.checkUTC("2009-02-01T01:00", "test");
+ ParamChecker.checkDateOozieTZ("2009-02-01T01:00", "test");
fail();
}
catch (Exception ex) {
}
try {
- ParamChecker.checkUTC("2009-02-01U01:00Z", "test");
+ ParamChecker.checkDateOozieTZ("2009-02-01U01:00Z", "test");
fail();
}
catch (Exception ex) {
Modified: incubator/oozie/trunk/core/src/test/resources/coord-action-for-action-input-check.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/resources/coord-action-for-action-input-check.xml?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/resources/coord-action-for-action-input-check.xml (original)
+++ incubator/oozie/trunk/core/src/test/resources/coord-action-for-action-input-check.xml Mon Aug 13 23:21:15 2012
@@ -15,7 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="1" timezone="UTC" freq_timeunit="DAY" end_of_duration="NONE" instance-number="1" action-nominal-time="2009-02-01T23:59Z" action-actual-time="2010-10-01T00:00Z">
+<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="1" timezone="UTC" freq_timeunit="DAY" end_of_duration="NONE" instance-number="1" action-nominal-time="2009-02-01T23:59${TZ}" action-actual-time="2010-10-01T00:00${TZ}">
<controls>
<timeout>10</timeout>
<concurrency>2</concurrency>
@@ -23,7 +23,7 @@
</controls>
<input-events>
<data-in name="A" dataset="a">
- <dataset name="a" frequency="7" initial-instance="2009-01-01T01:00Z" timezone="UTC" freq_timeunit="DAY" end_of_duration="NONE">
+ <dataset name="a" frequency="7" initial-instance="2009-01-01T01:00${TZ}" timezone="UTC" freq_timeunit="DAY" end_of_duration="NONE">
<uri-template>file://#testDir/${YEAR}/${DAY}</uri-template>
</dataset>
<uris>file://#testDir/2009/29#file://#testDir/2009/22#file://#testDir/2009/15#file://#testDir/2009/08</uris>
@@ -31,7 +31,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <dataset name="local_a" frequency="7" initial-instance="2009-01-01T01:00Z" timezone="UTC" freq_timeunit="DAY" end_of_duration="NONE">
+ <dataset name="local_a" frequency="7" initial-instance="2009-01-01T01:00${TZ}" timezone="UTC" freq_timeunit="DAY" end_of_duration="NONE">
<uri-template>file://#testDir/${YEAR}/${DAY}</uri-template>
</dataset>
<uris>file://#testDir/2009/29</uris>
Modified: incubator/oozie/trunk/docs/src/site/twiki/AG_Install.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/AG_Install.twiki?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/AG_Install.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/AG_Install.twiki Mon Aug 13 23:21:15 2012
@@ -396,6 +396,35 @@ streaming and pig actions.
The bundled Streaming and Pig JARs are the ones used by Oozie testcases.
+---++ Oozie Coordinators/Bundles Processing Timezone
+
+By default Oozie runs coordinator and bundle jobs using =UTC= timezone for datetime values specified in the application
+XML and in the job parameter properties. This includes coordinator applications start and end times of jobs, coordinator
+datasets initial-instance, bundle applications kick-offtimes. In addition, coordinator dataset instance URI templates
+will be resolved using datetime values of the Oozie processing timezone.
+
+It is possible to set the Oozie processing timezone to a timezone that is an offset of UTC, alternate timezones must
+expressed in using a GMT offset ( =GMT+/-####= ). For example: =GMT+0530= (India timezone).
+
+To change the default =UTC= timezone, use the =oozie.processing.timezone= property in the =oozie-site.xml=. For example:
+
+<verbatim>
+<configuration>
+ <property>
+ <name>oozie.processing.timezone</name>
+ <value>GMT+0530</value>
+ </property>
+</configuration>
+</verbatim>
+
+*IMPORTANT:* If using a processing timezone other than =UTC=, all datetime values in coordinator and bundle jobs must
+be expressed in the corresponding timezone, for example =2012-08-08T12:42+0530=.
+
+*NOTE:* It is strongly encouraged to use =UTC=, the default Oozie processing timezone.
+
+For more details on using an alternate Oozie processing timezone, please reffer to the
+[[CoordinatorFunctionalSpec#datetime][Coordinator Fuctional Specification, section '4. Datetime']]
+
---++ Advanced/Custom Environment Settings
Oozie can be configured to use Unix standard filesystem hierarchy for its different files
Modified: incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki Mon Aug 13 23:21:15 2012
@@ -125,12 +125,35 @@ Refer to section #6.5 'Parameterization
---++ 4. Datetime, Frequency and Time-Period Representation
+Oozie processes coordinator jobs in a fixed timezone with no DST (typically =UTC=), this timezone is referred as 'Oozie
+processing timezone'.
+
+The Oozie processing timezone is used to resolve coordinator jobs start/end times, job pause times and the initial-instance
+of datasets. Also, all coordinator dataset instance URI templates are resolved to a datetime in the Oozie processing
+time-zone.
+
+All the datetimes used in coordinator applications and job parameters to coordinator applications must be specified
+in the Oozie processing timezone. If Oozie processing timezone is =UTC=, the qualifier is *Z*. If Oozie processing
+time zone is other than =UTC=, the qualifier must be the GMT offset, =(+/-)####=.
+
+For example, a datetime in =UTC= is =2012-08-12T00:00Z=, the same datetime in =GMT+5:30= is =2012-08-12T05:30+0530=.
+
+For simplicity, there rest of this specification uses =UTC= datetimes.
+
+#datetime
---+++ 4.1. Datetime
-All datetime values are always in [[http://en.wikipedia.org/wiki/Coordinated_Universal_Time][UTC]] and [[http://www.w3.org/TR/NOTE-datetime][W3C Date-Time format]] down to a minute precision, 'YYYY-MM-DDTHH:mmZ'.
+If the Oozie processing timezone is =UTC=, all datetime values are always in
+[[http://en.wikipedia.org/wiki/Coordinated_Universal_Time][UTC]] down to a minute precision, 'YYYY-MM-DDTHH:mmZ'.
For example =2009-08-10T13:10Z= is August 10th 2009 at 13:10 UTC.
+If the Oozie processing timezone is a GMT offset =GMT(+/-)####=, all datetime values are always in
+[[http://en.wikipedia.org/wiki/ISO_8601][ISO 8601]] in the corresponding GMT offset down to a minute precision,
+'YYYY-MM-DDTHH:mmGMT(+/-)####'.
+
+For example =2009-08-10T13:10+0530= is August 10th 2009 at 13:10 GMT+0530, India timezone.
+
---++++ 4.1.1 End of the day in Datetime Values
It is valid to express the end of day as a '24:00' hour (i.e. =2009-08-10T24:00Z=).
@@ -150,7 +173,7 @@ Oozie Coordinator must provide a tool fo
---+++ 4.3. Timezones and Daylight-Saving
-While Oozie coordinator engine works in UTC, it provides DST support for coordinator applications.
+While Oozie coordinator engine works in a fixed timezone with no DST (typically =UTC=), it provides DST support for coordinator applications.
The baseline datetime for datasets and coordinator applications are expressed in UTC. The baseline datetime is the time of the first occurrence.
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1372663&r1=1372662&r2=1372663&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Mon Aug 13 23:21:15 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-948 Add support for Oozie coordinator to work in an UTC offset (tucu)
OOZIE-926 handling of global configuration is not correct (bcyr via tucu)
OOZIE-921 Changes in global section for the Name Node in FS action (bcyr via virag)
OOZIE-942 Add formal Parameters to bundle XML (rkanter via virag)