You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by vi...@apache.org on 2013/02/08 00:18:12 UTC
svn commit: r1443776 - in /oozie/branches/hcat-intre: ./
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/
tests/pig/src/test/java/org/apache/oozie/action/hadoop/
Author: virag
Date: Thu Feb 7 23:18:12 2013
New Revision: 1443776
URL: http://svn.apache.org/r1443776
Log:
Porting oozie-87 to hcat-intre and amending release-log.txt
Modified:
oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
oozie/branches/hcat-intre/release-log.txt
oozie/branches/hcat-intre/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
Modified: oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1443776&r1=1443775&r2=1443776&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java (original)
+++ oozie/branches/hcat-intre/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java Thu Feb 7 23:18:12 2013
@@ -479,14 +479,14 @@ public class JavaActionExecutor extends
// files and archives defined in the action
for (Element eProp : (List<Element>) actionXml.getChildren()) {
if (eProp.getName().equals("file")) {
- String[] pathes = eProp.getTextTrim().split(",");
- for (String path : pathes) {
+ String[] filePaths = eProp.getTextTrim().split(",");
+ for (String path : filePaths) {
addToCache(conf, appPath, path.trim(), false);
}
}
else if (eProp.getName().equals("archive")) {
- String[] pathes = eProp.getTextTrim().split(",");
- for (String path : pathes){
+ String[] archivePaths = eProp.getTextTrim().split(",");
+ for (String path : archivePaths){
addToCache(conf, appPath, path.trim(), true);
}
}
Modified: oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1443776&r1=1443775&r2=1443776&view=diff
==============================================================================
--- oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java (original)
+++ oozie/branches/hcat-intre/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java Thu Feb 7 23:18:12 2013
@@ -18,6 +18,7 @@
package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
@@ -46,6 +47,7 @@ import java.io.FileOutputStream;
import java.io.Writer;
import java.io.OutputStreamWriter;
import java.io.StringReader;
+import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
@@ -894,4 +896,115 @@ public class TestMapReduceActionExecutor
e.addContent(new Element("streaming"));
assertEquals("mapreduce-streaming", ae.getDefaultShareLibName(e));
}
+
+ /**
+ * https://issues.apache.org/jira/browse/OOZIE-87
+ * This test covers map-reduce action
+ * @throws Exception
+ */
+ public void testCommaSeparatedFilesAndArchives() throws Exception {
+ Path root = new Path(getFsTestCaseDir(), "root");
+
+ Path jar = new Path("jar.jar");
+ getFileSystem().create(new Path(getAppPath(), jar)).close();
+ Path rootJar = new Path(root, "rootJar.jar");
+ getFileSystem().create(rootJar).close();
+
+ Path file = new Path("file");
+ getFileSystem().create(new Path(getAppPath(), file)).close();
+ Path rootFile = new Path(root, "rootFile");
+ getFileSystem().create(rootFile).close();
+
+ Path so = new Path("soFile.so");
+ getFileSystem().create(new Path(getAppPath(), so)).close();
+ Path rootSo = new Path(root, "rootSoFile.so");
+ getFileSystem().create(rootSo).close();
+
+ Path so1 = new Path("soFile.so.1");
+ getFileSystem().create(new Path(getAppPath(), so1)).close();
+ Path rootSo1 = new Path(root, "rootSoFile.so.1");
+ getFileSystem().create(rootSo1).close();
+
+ Path archive = new Path("archive.tar");
+ getFileSystem().create(new Path(getAppPath(), archive)).close();
+ Path rootArchive = new Path(root, "rootArchive.tar");
+ getFileSystem().create(rootArchive).close();
+
+ String actionXml = "<map-reduce>" +
+ " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ " <name-node>" + getNameNodeUri() + "</name-node>" +
+ " <main-class>CLASS</main-class>" +
+ " <file>" + jar.toString() +
+ "," + rootJar.toString() +
+ "," + file.toString() +
+ ", " + rootFile.toString() + // with leading and trailing spaces
+ " ," + so.toString() +
+ "," + rootSo.toString() +
+ "," + so1.toString() +
+ "," + rootSo1.toString() + "</file>\n" +
+ " <archive>" + archive.toString() + ", "
+ + rootArchive.toString() + " </archive>\n" + // with leading and trailing spaces
+ "</map-reduce>";
+
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+
+ Context context = createContext("map-reduce", actionXml);
+
+ Path appPath = getAppPath();
+
+ MapReduceActionExecutor ae = new MapReduceActionExecutor();
+
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setupActionConf(jobConf, context, eActionXml, appPath);
+ ae.setLibFilesArchives(context, eActionXml, appPath, jobConf);
+
+
+ assertTrue(DistributedCache.getSymlink(jobConf));
+
+ Path[] filesInClasspath = DistributedCache.getFileClassPaths(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar}) {
+ boolean found = false;
+ for (Path c : filesInClasspath) {
+ if (!found && p.toUri().getPath().equals(c.toUri().getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("file " + p.toUri().getPath() + " not found in classpath", found);
+ }
+ for (Path p : new Path[]{new Path(getAppPath(), file), rootFile, new Path(getAppPath(), so), rootSo,
+ new Path(getAppPath(), so1), rootSo1}) {
+ boolean found = false;
+ for (Path c : filesInClasspath) {
+ if (!found && p.toUri().getPath().equals(c.toUri().getPath())) {
+ found = true;
+ }
+ }
+ assertFalse("file " + p.toUri().getPath() + " found in classpath", found);
+ }
+
+ URI[] filesInCache = DistributedCache.getCacheFiles(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar, new Path(getAppPath(), file), rootFile,
+ new Path(getAppPath(), so), rootSo, new Path(getAppPath(), so1), rootSo1}) {
+ boolean found = false;
+ for (URI c : filesInCache) {
+ if (!found && p.toUri().getPath().equals(c.getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("file " + p.toUri().getPath() + " not found in cache", found);
+ }
+
+ URI[] archivesInCache = DistributedCache.getCacheArchives(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), archive), rootArchive}) {
+ boolean found = false;
+ for (URI c : archivesInCache) {
+ if (!found && p.toUri().getPath().equals(c.getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("archive " + p.toUri().getPath() + " not found in cache", found);
+ }
+ }
+
+
}
Modified: oozie/branches/hcat-intre/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/release-log.txt?rev=1443776&r1=1443775&r2=1443776&view=diff
==============================================================================
--- oozie/branches/hcat-intre/release-log.txt (original)
+++ oozie/branches/hcat-intre/release-log.txt Thu Feb 7 23:18:12 2013
@@ -1,10 +1,16 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via rkanter)
OOZIE-809 MySQL TEXT columns should be MEDIUMTEXT (rkanter via tucu)
OOZIE-669 Deprecate oozie-start.sh, oozie-stop.sh & oozie-run.sh scripts (rkanter via tucu)
+OOZIE-1015 HadoopAccessorService jobtracker validation should not have hardcoded conf key (mona)
+OOZIE-1078 Help -> Documentation and Help -> Online Help should link to oozie.apache.org/ (jun via mohammad)
+OOZIE-979 bump up trunk version to 3.4.0-SNAPSHOT (tucu)
+OOZIE-976 add workflowgenerator into distro tarball (egashira via tucu)
+OOZIE-944 Implement Workflow Generator UI Tool (egashira via virag)
--- Oozie 3.2.0 release
+-- Oozie 3.3.2 unreleased
OOZIE-1196 HCat EL functions for database and table should be modified (mona)
OOZIE-1210 Rework uri handling for Prepare actions and jms server mapping (rohini via virag)
@@ -14,6 +20,7 @@ OOZIE-1177 HostnameFilter should only ca
OOZIE-1194 test-patch shouldn't run the testHive profile because it not longer exists (rkanter)
OOZIE-1193 upgrade jython to 2.5.3 for Pig in Oozie due to jython 2.5.0 legal issues (bowenzhangusa via rkanter)
OOZIE-1172 Add documentation on how to get Java actions to authenticate properly on Kerberos-enabled clusters (rkanter)
+OOZIE-87 GH-47: Feature to supply a comma separated list of jars in an 'archive tag' of workflow (jaoki via rkanter)
OOZIE-945 BundleSubmitXCommand.submit() doesn't properly remove comments (jaoki via rkanter)
OOZIE-1171 HostnameFilter should handle hostname resolution failures and continue processing (tucu via rkanter)
OOZIE-1166 Print a more helpful message when ProxyUserService is configured wrong (rkanter)
@@ -22,6 +29,7 @@ OOZIE-1051 Repeating Errors for workflow
OOZIE-1140 TestLogStreamer.testStreamLog fails when its started within the first 4 seconds after the hour (rkanter)
OOZIE-1153 comma separated list in <archive> and <file> for JavaActionExecutor. (jaoki via tucu)
OOZIE-1160 Oozie web-console to display all job URLs spawned by Pig (mona)
+OOZIE-1164 typo in toString() method for org.apache.oozie.client.rest.JsonCoordinatorJob.java (bowenzhangusa via rkanter)
OOZIE-1186 Image load for Job DAG visualization should handle resources better (mona)
OOZIE-1185 Retry jms connections on failure (rohini via virag)
OOZIE-1158 Add hcataloglib sub-module (mona)
@@ -32,7 +40,6 @@ OOZIE-1053 Oozie Web-console clicking on
OOZIE-1161 Remove unnecessary db updates for some of the blobs like missing_dependencies' of Coordinator Action(virag)
OOZIE-1156 Make all the latest/future instances as pull dependences (virag)
OOZIE-1144 OOZIE-1137 breaks the sharelib (rkanter)
-OOZIE-1035 Improve forkjoin validation to allow same errorTo transitions (rkanter)
OOZIE-1137 In light of federation use actionLibPath instead of appPath (vaidya via rkanter)
OOZIE-1102 Update Oozie README.txt to have the TLP mailing list and links (jaoki via rkanter)
OOZIE-1103 Create example using AltKerberosAuthenticationHandler (rkanter)
@@ -55,15 +62,13 @@ OOZIE-1111 change HCatURI to specify par
OOZIE-1108 Fix JMS message consumer to maintain single session per topic registration (mona)
OOZIE-1075 Create general scheme handler (rohini via virag)
OOZIE-1071 latest EL function is based on action materialization time (rohini via virag)
+ OOZIE-1035 Improve forkjoin validation to allow same errorTo transitions (rkanter)
OOZIE-1114 Some tests don't use the Services singleton properly (rkanter)
OOZIE-1106 latest and future function do not work correctly when oozie processing timezone is non UTC (rohini via tucu)
OOZIE-1073 Optimize latest and future EL resolution in case of start-instance and end-instance (rohini via virag)
OOZIE-1107 Change default done-flag from _SUCCESS to empty for Hcat (mohammad)
OOZIE-1095 Add HCatalog jar as resource for building (mona)
OOZIE-1105 Resolve issues found during integration(mohammad)
-OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via rkanter)
-OOZIE-1015 HadoopAccessorService jobtracker validation should not have hardcoded conf key (mona)
-OOZIE-1078 Help -> Documentation and Help -> Online Help should link to oozie.apache.org/ (jun via mohammad)
OOZIE-1057 Log message for retrying to connect to the JT always says 60,000 milliseconds (jiezhou via rkanter)
OOZIE-1080 Add a dryrun option for workflows (rkanter)
OOZIE-1086 Command to check the missing partitions directly against HCatalog server (mohammad)
@@ -116,7 +121,6 @@ OOZIE-984 Allow EL Functions in Coordina
OOZIE-972 Provide EL function to append a string in each substring of another string separated by delimiter (Mohammad via virag)
OOZIE-977 NotificationXCommand (job.notification queue entry) should set a timeout in the HTTP connections it makes (tucu)
OOZIE-654 Provide a way to use 'uber' jars with Oozie MR actions (rkanter via tucu)
-OOZIE-979 bump up trunk version to 3.4.0-SNAPSHOT (tucu)
-- Oozie 3.3.1 (unreleased)
@@ -143,7 +147,6 @@ OOZIE-1100 HFTP coordinator input check
OOZIE-1058 ACL modify-job should not be hardcoded to group name(mona via mohammad)
OOZIE-1052 HadoopAccessorService.createFileSystem throws exception in map-reduce action, failing workflow (egashira via mohammad)
-OOZIE-976 add workflowgenerator into distro tarball (egashira via tucu)
OOZIE-993 Hadoop 23 doesn't accept user defined jobtracker (virag)
OOZIE-1005 Tests from OOZIE-994 use wrong condition in waitFor (rkanter via virag)
OOZIE-994 ActionCheckXCommand does not handle failures properly (rkanter via virag)
@@ -161,7 +164,6 @@ OOZIE-971 TestRecoveryService failing ve
OOZIE-966 Fix formatting in CLI output when GMT-#### and GMT-##:## formatted timezones are used (rkanter via tucu)
OOZIE-960 TestStatusTransitService failing intermittently (virag)
OOZIE-968 source oozie environment from conf in oozie db setup script (svenkat via virag)
-OOZIE-944 Implement Workflow Generator UI Tool (egashira via virag)
OOZIE-961 Load Hbase credentials in Oozie (virag)
OOZIE-963 Add new EL function to replace all instances of a sub-string with another one (Mohammad via virag)
OOZIE-969 Unit tests in TestStatusTransitService failing due to change in CoordKillX (mona via virag)
Modified: oozie/branches/hcat-intre/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/hcat-intre/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java?rev=1443776&r1=1443775&r2=1443776&view=diff
==============================================================================
--- oozie/branches/hcat-intre/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java (original)
+++ oozie/branches/hcat-intre/tests/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java Thu Feb 7 23:18:12 2013
@@ -18,6 +18,7 @@
package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.mapred.JobConf;
@@ -26,6 +27,7 @@ import org.apache.hadoop.mapred.RunningJ
import org.apache.hadoop.mapred.JobID;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.action.hadoop.ActionExecutorTestCase.Context;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
@@ -45,6 +47,7 @@ import java.io.FileInputStream;
import java.io.Writer;
import java.io.OutputStreamWriter;
import java.io.StringReader;
+import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -420,4 +423,112 @@ public class TestPigActionExecutor exten
_testSubmit(actionXml, true);
}
+ /**
+ * https://issues.apache.org/jira/browse/OOZIE-87
+ * This test covers pig action
+ * @throws Exception
+ */
+ public void testCommaSeparatedFilesAndArchives() throws Exception {
+ Path root = new Path(getFsTestCaseDir(), "root");
+
+ Path jar = new Path("jar.jar");
+ getFileSystem().create(new Path(getAppPath(), jar)).close();
+ Path rootJar = new Path(root, "rootJar.jar");
+ getFileSystem().create(rootJar).close();
+
+ Path file = new Path("file");
+ getFileSystem().create(new Path(getAppPath(), file)).close();
+ Path rootFile = new Path(root, "rootFile");
+ getFileSystem().create(rootFile).close();
+
+ Path so = new Path("soFile.so");
+ getFileSystem().create(new Path(getAppPath(), so)).close();
+ Path rootSo = new Path(root, "rootSoFile.so");
+ getFileSystem().create(rootSo).close();
+
+ Path so1 = new Path("soFile.so.1");
+ getFileSystem().create(new Path(getAppPath(), so1)).close();
+ Path rootSo1 = new Path(root, "rootSoFile.so.1");
+ getFileSystem().create(rootSo1).close();
+
+ Path archive = new Path("archive.tar");
+ getFileSystem().create(new Path(getAppPath(), archive)).close();
+ Path rootArchive = new Path(root, "rootArchive.tar");
+ getFileSystem().create(rootArchive).close();
+
+ String actionXml = "<pig>" +
+ " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ " <name-node>" + getNameNodeUri() + "</name-node>" +
+ " <script>id.pig</script>" +
+ " <file>" + jar.toString() +
+ "," + rootJar.toString() +
+ "," + file.toString() +
+ ", " + rootFile.toString() + // with leading and trailing spaces
+ " ," + so.toString() +
+ "," + rootSo.toString() +
+ "," + so1.toString() +
+ "," + rootSo1.toString() + "</file>\n" +
+ " <archive>" + archive.toString() + ", "
+ + rootArchive.toString() + " </archive>\n" + // with leading and trailing spaces
+ "</pig>";
+
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+
+ Context context = createContext(actionXml);
+
+ Path appPath = getAppPath();
+
+ PigActionExecutor ae = new PigActionExecutor();
+
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setupActionConf(jobConf, context, eActionXml, appPath);
+ ae.setLibFilesArchives(context, eActionXml, appPath, jobConf);
+
+
+ assertTrue(DistributedCache.getSymlink(jobConf));
+
+ Path[] filesInClasspath = DistributedCache.getFileClassPaths(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar}) {
+ boolean found = false;
+ for (Path c : filesInClasspath) {
+ if (!found && p.toUri().getPath().equals(c.toUri().getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("file " + p.toUri().getPath() + " not found in classpath", found);
+ }
+ for (Path p : new Path[]{new Path(getAppPath(), file), rootFile, new Path(getAppPath(), so), rootSo,
+ new Path(getAppPath(), so1), rootSo1}) {
+ boolean found = false;
+ for (Path c : filesInClasspath) {
+ if (!found && p.toUri().getPath().equals(c.toUri().getPath())) {
+ found = true;
+ }
+ }
+ assertFalse("file " + p.toUri().getPath() + " found in classpath", found);
+ }
+
+ URI[] filesInCache = DistributedCache.getCacheFiles(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), jar), rootJar, new Path(getAppPath(), file), rootFile,
+ new Path(getAppPath(), so), rootSo, new Path(getAppPath(), so1), rootSo1}) {
+ boolean found = false;
+ for (URI c : filesInCache) {
+ if (!found && p.toUri().getPath().equals(c.getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("file " + p.toUri().getPath() + " not found in cache", found);
+ }
+
+ URI[] archivesInCache = DistributedCache.getCacheArchives(jobConf);
+ for (Path p : new Path[]{new Path(getAppPath(), archive), rootArchive}) {
+ boolean found = false;
+ for (URI c : archivesInCache) {
+ if (!found && p.toUri().getPath().equals(c.getPath())) {
+ found = true;
+ }
+ }
+ assertTrue("archive " + p.toUri().getPath() + " not found in cache", found);
+ }
+ }
}