You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/08/06 19:56:07 UTC
svn commit: r1369898 - in /incubator/oozie/trunk: ./
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/
Author: tucu
Date: Mon Aug 6 17:56:07 2012
New Revision: 1369898
URL: http://svn.apache.org/viewvc?rev=1369898&view=rev
Log:
OOZIE-938 Remove some duplicated code in FsActionExecutor (rkanter via tucu)
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java?rev=1369898&r1=1369897&r2=1369898&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java Mon Aug 6 17:56:07 2012
@@ -117,45 +117,6 @@ public class FsActionExecutor extends Ac
"move, target NN URI different from that of source", dest);
}
}
-
- XConfiguration parseJobXML(Context context, Element element)
- throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
- Path appPathRoot = new Path(context.getWorkflow().getAppPath());
-
- FileSystem fs = context.getAppFileSystem();
-
- // app path could be a file
- if (context.getAppFileSystem().isFile(appPathRoot)) {
- appPathRoot = appPathRoot.getParent();
- }
-
- XConfiguration jobXmlConfs = null;
- Iterator<Element> it = element.getChildren("job-xml", element.getNamespace()).iterator();
- while (it.hasNext()) {
- Element e = it.next();
- String jobXml = e.getTextTrim();
- Path path = new Path(appPathRoot, jobXml);
- XConfiguration jobXmlConf = new XConfiguration(fs.open(path));
- JavaActionExecutor.checkForDisallowedProps(jobXmlConf, "job-xml");
- if (jobXmlConfs == null) {
- jobXmlConfs = new XConfiguration();
- }
- XConfiguration.copy(jobXmlConf, jobXmlConfs);
- }
- return jobXmlConfs;
- }
-
- XConfiguration parseConfiguration(Element element)
- throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
- Element e = element.getChild("configuration", element.getNamespace());
- if (e != null) {
- String strConf = XmlUtils.prettyPrint(e).toString();
- XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
- JavaActionExecutor.checkForDisallowedProps(inlineConf, "inline configuration");
- return inlineConf;
- }
- return null;
- }
@SuppressWarnings("unchecked")
void doOperations(Context context, Element element) throws ActionExecutorException {
@@ -178,14 +139,12 @@ public class FsActionExecutor extends Ac
}
XConfiguration fsConf = new XConfiguration();
- XConfiguration jobXMLConf = parseJobXML(context, element);
- if (jobXMLConf != null) {
- XConfiguration.copy(jobXMLConf, fsConf);
- }
- XConfiguration configConf = parseConfiguration(element);
- if (configConf != null) {
- XConfiguration.copy(configConf, fsConf);
+ Path appPath = new Path(context.getWorkflow().getAppPath());
+ // app path could be a file
+ if (fs.isFile(appPath)) {
+ appPath = appPath.getParent();
}
+ JavaActionExecutor.parseJobXmlAndConfiguration(context, element, appPath, fsConf);
for (Element commandElement : (List<Element>) element.getChildren()) {
String command = commandElement.getName();
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1369898&r1=1369897&r2=1369898&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java Mon Aug 6 17:56:07 2012
@@ -234,55 +234,47 @@ public class JavaActionExecutor extends
throw convertException(ex);
}
}
-
- protected FileSystem getActionFileSystem(Context context, WorkflowAction action) throws ActionExecutorException {
- try {
- Element actionXml = XmlUtils.parseXml(action.getConf());
- return getActionFileSystem(context, actionXml);
- }
- catch (JDOMException ex) {
- throw convertException(ex);
- }
- }
-
- protected FileSystem getActionFileSystem(Context context, Element actionXml) throws ActionExecutorException {
- try {
- return context.getAppFileSystem();
- }
- catch (Exception ex) {
- throw convertException(ex);
+
+ public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf)
+ throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
+ Namespace ns = element.getNamespace();
+ Iterator<Element> it = element.getChildren("job-xml", ns).iterator();
+ while (it.hasNext()) {
+ Element e = it.next();
+ String jobXml = e.getTextTrim();
+ Path path = new Path(appPath, jobXml);
+ FileSystem fs = context.getAppFileSystem();
+ Configuration jobXmlConf = new XConfiguration(fs.open(path));
+ checkForDisallowedProps(jobXmlConf, "job-xml");
+ XConfiguration.copy(jobXmlConf, conf);
+ }
+ Element e = element.getChild("configuration", ns);
+ if (e != null) {
+ String strConf = XmlUtils.prettyPrint(e).toString();
+ XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
+ checkForDisallowedProps(inlineConf, "inline configuration");
+ XConfiguration.copy(inlineConf, conf);
}
}
-
- Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
+
+ Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
throws ActionExecutorException {
try {
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_JOB_TRACKER), getType());
XConfiguration.injectDefaults(actionDefaults, actionConf);
- Namespace ns = actionXml.getNamespace();
- Iterator<Element> it = actionXml.getChildren("job-xml", ns).iterator();
- while (it.hasNext()) {
- Element e = it.next();
- String jobXml = e.getTextTrim();
- Path path = new Path(appPath, jobXml);
- FileSystem fs = getActionFileSystem(context, actionXml);
- Configuration jobXmlConf = new XConfiguration(fs.open(path));
- checkForDisallowedProps(jobXmlConf, "job-xml");
- XConfiguration.copy(jobXmlConf, actionConf);
- }
- Element e = actionXml.getChild("configuration", ns);
- if (e != null) {
- String strConf = XmlUtils.prettyPrint(e).toString();
- XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
- checkForDisallowedProps(inlineConf, "inline configuration");
- XConfiguration.copy(inlineConf, actionConf);
- }
+ parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf);
return actionConf;
}
catch (IOException ex) {
throw convertException(ex);
}
+ catch (HadoopAccessorException ex) {
+ throw convertException(ex);
+ }
+ catch (URISyntaxException ex) {
+ throw convertException(ex);
+ }
}
Configuration addToCache(Configuration conf, Path appPath, String filePath, boolean archive)
@@ -868,7 +860,7 @@ public class JavaActionExecutor extends
public void start(Context context, WorkflowAction action) throws ActionExecutorException {
try {
XLog.getLog(getClass()).debug("Starting action " + action.getId() + " getting Action File System");
- FileSystem actionFs = getActionFileSystem(context, action);
+ FileSystem actionFs = context.getAppFileSystem();
XLog.getLog(getClass()).debug("Preparing action Dir through copying " + context.getActionDir());
prepareActionDir(actionFs, context);
XLog.getLog(getClass()).debug("Action Dir is ready. Submitting the action ");
@@ -895,7 +887,7 @@ public class JavaActionExecutor extends
}
finally {
try {
- FileSystem actionFs = getActionFileSystem(context, action);
+ FileSystem actionFs = context.getAppFileSystem();
cleanUpActionDir(actionFs, context);
}
catch (Exception ex) {
@@ -924,7 +916,7 @@ public class JavaActionExecutor extends
boolean exception = false;
try {
Element actionXml = XmlUtils.parseXml(action.getConf());
- FileSystem actionFs = getActionFileSystem(context, actionXml);
+ FileSystem actionFs = context.getAppFileSystem();
JobConf jobConf = createBaseHadoopConf(context, actionXml);
jobClient = createJobClient(context, jobConf);
RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId()));
@@ -1089,7 +1081,7 @@ public class JavaActionExecutor extends
}
finally {
try {
- FileSystem actionFs = getActionFileSystem(context, action);
+ FileSystem actionFs = context.getAppFileSystem();
cleanUpActionDir(actionFs, context);
if (jobClient != null) {
jobClient.close();
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java?rev=1369898&r1=1369897&r2=1369898&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java Mon Aug 6 17:56:07 2012
@@ -174,78 +174,6 @@ public class TestFsActionExecutor extend
assertEquals("FS007", ex.getErrorCode());
}
}
-
- public void testParseJobXML() throws Exception {
- FsActionExecutor ae = new FsActionExecutor();
- FileSystem fs = getFileSystem();
-
- Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
-
- String str = MessageFormat.format("<fs>"
- + "<mkdir path=''{0}''/>"
- + "</fs>", mkdir);
- Element xml = XmlUtils.parseXml(str);
-
- assertNull(ae.parseJobXML(createContext("<fs/>"), xml));
-
-
- str = MessageFormat.format("<fs>"
- + "<job-xml>job1.xml</job-xml>"
- + "<job-xml>job2.xml</job-xml>"
- + "<mkdir path=''{0}''/>"
- + "</fs>", mkdir);
- xml = XmlUtils.parseXml(str);
-
- Path appPath = new Path(getFsTestCaseDir(), "app");
- getFileSystem().mkdirs(appPath);
-
- XConfiguration jConf = new XConfiguration();
- jConf.set("p1", "v1");
- jConf.set("p2", "v2");
- OutputStream os = getFileSystem().create(new Path(appPath, "job1.xml"));
- jConf.writeXml(os);
- os.close();
-
- jConf = new XConfiguration();
- jConf.set("p3", "v3");
- jConf.set("p2", "v4");
- os = getFileSystem().create(new Path(appPath, "job2.xml"));
- jConf.writeXml(os);
- os.close();
-
- XConfiguration jobXMLConf = ae.parseJobXML(createContext("<fs/>"), xml);
- assertEquals(jobXMLConf.get("p1"), "v1");
- assertEquals(jobXMLConf.get("p2"), "v4");
- assertEquals(jobXMLConf.get("p3"), "v3");
- }
-
- public void testParseConfiguration() throws Exception {
- FsActionExecutor ae = new FsActionExecutor();
- FileSystem fs = getFileSystem();
-
- Path mkdir = new Path(getFsTestCaseDir(), "mkdir");
-
- String str = MessageFormat.format("<fs>"
- + "<mkdir path=''{0}''/>"
- + "</fs>", mkdir);
- Element xml = XmlUtils.parseXml(str);
-
- assertNull(ae.parseConfiguration(xml));
-
-
- str = MessageFormat.format("<fs>"
- + "<configuration>"
- + "<property><name>p1</name><value>v1</value></property>"
- + "<property><name>p2</name><value>v2</value></property>"
- + "</configuration>"
- + "<mkdir path=''{0}''/>"
- + "</fs>", mkdir);
- xml = XmlUtils.parseXml(str);
-
- XConfiguration configConf = ae.parseConfiguration(xml);
- assertEquals(configConf.get("p1"), "v1");
- assertEquals(configConf.get("p2"), "v2");
- }
public void testMkdir() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1369898&r1=1369897&r2=1369898&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java Mon Aug 6 17:56:07 2012
@@ -1076,4 +1076,41 @@ public class TestJavaActionExecutor exte
assertNotSame(conf.get(JavaActionExecutor.ACL_VIEW_JOB), actionConf.get(JavaActionExecutor.ACL_VIEW_JOB));
assertNotSame(conf.get(JavaActionExecutor.ACL_MODIFY_JOB), actionConf.get(JavaActionExecutor.ACL_MODIFY_JOB));
}
+
+ public void testParseJobXmlAndConfiguration() throws Exception {
+ String str = "<java>"
+ + "<job-xml>job1.xml</job-xml>"
+ + "<job-xml>job2.xml</job-xml>"
+ + "<configuration>"
+ + "<property><name>p1</name><value>v1a</value></property>"
+ + "<property><name>p2</name><value>v2</value></property>"
+ + "</configuration>"
+ + "</java>";
+ Element xml = XmlUtils.parseXml(str);
+ Path appPath = new Path(getFsTestCaseDir(), "app");
+ getFileSystem().mkdirs(appPath);
+
+ XConfiguration jConf = new XConfiguration();
+ jConf.set("p1", "v1b");
+ jConf.set("p3", "v3a");
+ OutputStream os = getFileSystem().create(new Path(appPath, "job1.xml"));
+ jConf.writeXml(os);
+ os.close();
+
+ jConf = new XConfiguration();
+ jConf.set("p4", "v4");
+ jConf.set("p3", "v3b");
+ os = getFileSystem().create(new Path(appPath, "job2.xml"));
+ jConf.writeXml(os);
+ os.close();
+
+ Configuration conf = new XConfiguration();
+ assertEquals(0, conf.size());
+ JavaActionExecutor.parseJobXmlAndConfiguration(createContext("<java/>"), xml, appPath, conf);
+ assertEquals(4, conf.size());
+ assertEquals("v1a", conf.get("p1"));
+ assertEquals("v2", conf.get("p2"));
+ assertEquals("v3b", conf.get("p3"));
+ assertEquals("v4", conf.get("p4"));
+ }
}
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1369898&r1=1369897&r2=1369898&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Mon Aug 6 17:56:07 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-938 Remove some duplicated code in FsActionExecutor (rkanter via tucu)
OOZIE-939 JT_PRINCIPAL and NN_PRINCIPAL must be added back to XOozieClient (rkanter via tucu)
OOZIE-930 Bundle not pass SUSPEND command to PAUSED coord job (virag)
OOZIE-933 Modify the Oozie CLI help to escape the semi-colon when multiple filters are specified (virag)