You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2013/02/12 00:13:46 UTC
svn commit: r1444991 - in /oozie/branches/hcat-intre:
core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
release-log.txt
Author: virag
Date: Mon Feb 11 23:13:45 2013
New Revision: 1444991
URL: http://svn.apache.org/r1444991
Log:
OOZIE-1211 oozie does not support duplicated dataset (jaoki via virag)
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
oozie/branches/hcat-intre/release-log.txt
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java?rev=1444991&r1=1444990&r2=1444991&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java Mon Feb 11 23:13:45 2013
@@ -948,7 +948,7 @@ public class CoordSubmitXCommand extends
for (Element e : (List<Element>) datasets.getChildren("dataset", datasets.getNamespace())) {
String dsName = e.getAttributeValue("name");
if (dsList.contains(dsName)) {// Override with this DS
- // Remove old DS
+ // Remove duplicate
removeDataSet(allDataSets, dsName);
}
else {
@@ -1017,6 +1017,7 @@ public class CoordSubmitXCommand extends
for (Element eDataset : (List<Element>) eDatasets.getChildren("dataset", eDatasets.getNamespace())) {
if (eDataset.getAttributeValue("name").equals(name)) {
eDataset.detach();
+ return;
}
}
throw new RuntimeException("undefined dataset: " + name);
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java?rev=1444991&r1=1444990&r2=1444991&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java Mon Feb 11 23:13:45 2013
@@ -24,6 +24,7 @@ import java.io.PrintWriter;
import java.io.Reader;
import java.io.Writer;
import java.net.URI;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.BundleJobBean;
@@ -39,6 +40,9 @@ import org.apache.oozie.service.Services
import org.apache.oozie.test.XDataTestCase;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.jdom.Element;
+import org.jdom.Namespace;
public class TestCoordSubmitXCommand extends XDataTestCase {
@@ -779,6 +783,142 @@ public class TestCoordSubmitXCommand ext
}
}
+ /**
+ * Basic submit with include file
+ * @throws Exception
+ */
+ public void testBasicSubmitWithIncludeFile() throws Exception {
+ Configuration conf = new XConfiguration();
+ final String includePath = "file://" + getTestCaseDir() + File.separator + "include1.xml";
+ final String URI_TEMPLATE_INCLUDE_XML = "file:///tmp/include_xml/workflows/${YEAR}/${DAY}";
+ final String URI_TEMPLATE_COORD_XML = "file:///tmp/coord_xml/workflows/${YEAR}/${DAY}";
+ String includeXml =
+ "<datasets> "
+ + "<dataset name=\"A\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">"
+ + "<uri-template>" + URI_TEMPLATE_INCLUDE_XML + "</uri-template>"
+ + "</dataset> "
+ + "</datasets>";
+ writeToFile(includeXml, includePath);
+
+ String appPath = "file://" + getTestCaseDir() + File.separator + "coordinator.xml";
+ String appXml =
+ "<coordinator-app name=\"${appName}-foo\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" "
+ + "end=\"2009-02-03T23:59Z\" timezone=\"UTC\" xmlns=\"uri:oozie:coordinator:0.2\">"
+ + "<controls> "
+ + "<execution>LIFO</execution>"
+ + "</controls>"
+ + "<datasets> "
+ + "<include>" + includePath + "</include>"
+ + "<dataset name=\"B\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">"
+ + "<uri-template>" + URI_TEMPLATE_COORD_XML + "</uri-template>"
+ + "</dataset> "
+ + "</datasets>"
+ + " <input-events> "
+ + "<data-in name=\"inputA\" dataset=\"A\"> <instance>${coord:latest(0)}</instance> </data-in> "
+ + "<data-in name=\"inputB\" dataset=\"B\"> <instance>${coord:latest(0)}</instance> </data-in> "
+ + "</input-events> "
+ + "<action>"
+ + "<workflow>"
+ + "<app-path>hdfs:///tmp/workflows/</app-path> "
+ + "<configuration>"
+ + "<property> <name>inputA</name> <value>${coord:dataIn('inputB')}</value> </property> "
+ + "</configuration>"
+ + "</workflow>"
+ + "</action>"
+ + " </coordinator-app>";
+ writeToFile(appXml, appPath);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("appName", "var-app-name");
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf, "UNIT_TESTING");
+ String jobId = sc.call();
+
+ assertEquals(jobId.substring(jobId.length() - 2), "-C");
+ CoordinatorJobBean job = checkCoordJobs(jobId);
+ assertNotNull(job);
+ Element processedJobXml = XmlUtils.parseXml(job.getJobXml());
+
+ Namespace namespace = processedJobXml.getNamespace();
+ @SuppressWarnings("unchecked")
+ List<Element> datainElements = processedJobXml.getChild("input-events", namespace).getChildren("data-in", namespace);
+ assertTrue("<data-in> should be 2. One from coordinator.xml and the other from the include file"
+ , datainElements.size() == 2);
+
+ assertEquals(URI_TEMPLATE_INCLUDE_XML
+ , datainElements.get(0).getChild("dataset", namespace).getChildText("uri-template", namespace));
+ assertEquals(URI_TEMPLATE_COORD_XML
+ , datainElements.get(1).getChild("dataset", namespace).getChildText("uri-template", namespace));
+ }
+
+ /**
+ * https://issues.apache.org/jira/browse/OOZIE-1211
+ * If a datasets include file has a dataset name as in one defined in coordinator.xml,
+ * the one in coordinator.xml should be honored.
+ * http://oozie.apache.org/docs/3.3.1/CoordinatorFunctionalSpec.html#a10.1.1._Dataset_Names_Collision_Resolution
+ *
+ * @throws Exception
+ */
+ public void testDuplicateDatasetNameInIncludeFile() throws Exception {
+ Configuration conf = new XConfiguration();
+ final String includePath = "file://" + getTestCaseDir() + File.separator + "include1.xml";
+ final String URI_TEMPLATE_INCLUDE_XML = "file:///tmp/include_xml/workflows/${YEAR}/${DAY}";
+ final String URI_TEMPLATE_COORD_XML = "file:///tmp/coord_xml/workflows/${YEAR}/${DAY}";
+ String includeXml =
+ "<datasets> "
+ + "<dataset name=\"B\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">"
+ + "<uri-template>" + URI_TEMPLATE_INCLUDE_XML + "</uri-template>"
+ + "</dataset> "
+ + "</datasets>";
+ writeToFile(includeXml, includePath);
+
+ String appPath = "file://" + getTestCaseDir() + File.separator + "coordinator.xml";
+ String appXml =
+ "<coordinator-app name=\"${appName}-foo\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" "
+ + "end=\"2009-02-03T23:59Z\" timezone=\"UTC\" xmlns=\"uri:oozie:coordinator:0.2\">"
+ + "<controls> "
+ + "<execution>LIFO</execution>"
+ + "</controls>"
+ + "<datasets> "
+ + "<include>" + includePath + "</include>"
+ + "<dataset name=\"B\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" timezone=\"UTC\">"
+ + "<uri-template>" + URI_TEMPLATE_COORD_XML + "</uri-template>"
+ + "</dataset> "
+ + "</datasets>"
+ + " <input-events> "
+ + "<data-in name=\"inputB\" dataset=\"B\"> <instance>${coord:latest(0)}</instance> </data-in> "
+ + "</input-events> "
+ + "<action>"
+ + "<workflow>"
+ + "<app-path>hdfs:///tmp/workflows/</app-path> "
+ + "<configuration>"
+ + "<property> <name>inputB</name> <value>${coord:dataIn('inputB')}</value> </property> "
+ + "</configuration>"
+ + "</workflow>"
+ + "</action>"
+ + " </coordinator-app>";
+ writeToFile(appXml, appPath);
+ conf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
+ conf.set(OozieClient.USER_NAME, getTestUser());
+ conf.set("appName", "var-app-name");
+ CoordSubmitXCommand sc = new CoordSubmitXCommand(conf, "UNIT_TESTING");
+ String jobId = sc.call();
+
+ assertEquals(jobId.substring(jobId.length() - 2), "-C");
+ CoordinatorJobBean job = checkCoordJobs(jobId);
+ assertNotNull(job);
+ Element processedJobXml = XmlUtils.parseXml(job.getJobXml());
+
+ Namespace namespace = processedJobXml.getNamespace();
+ @SuppressWarnings("unchecked")
+ List<Element> datasetElements = processedJobXml.getChild("input-events", namespace).getChild("data-in", namespace)
+ .getChildren("dataset", namespace);
+ assertTrue("<dataset> should not be duplicate", datasetElements.size() == 1);
+
+ assertEquals(URI_TEMPLATE_COORD_XML, datasetElements.get(0).getChildText("uri-template", namespace));
+ assertFalse("<uri-template> should not contain one from the include file"
+ , job.getJobXml().contains(URI_TEMPLATE_INCLUDE_XML));
+ }
+
private void _testConfigDefaults(boolean withDefaults) throws Exception {
Configuration conf = new XConfiguration();
String appPath = "file://" + getTestCaseDir() + File.separator + "coordinator.xml";
Modified: oozie/branches/hcat-intre/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1444991&r1=1444990&r2=1444991&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Mon Feb 11 23:13:45 2013
@@ -1,6 +1,7 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1211 oozie does not support duplicated dataset (jaoki via virag)
OOZIE-1187 reduce memory usage of SLA query (invoked by CLI command) to avoid OOM (egashira via virag)
OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via rkanter)
OOZIE-809 MySQL TEXT columns should be MEDIUMTEXT (rkanter via tucu)