You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/09/12 22:35:26 UTC
svn commit: r1384113 - in /oozie/branches/branch-3.3: ./
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/
Author: tucu
Date: Wed Sep 12 20:35:25 2012
New Revision: 1384113
URL: http://svn.apache.org/viewvc?rev=1384113&view=rev
Log:
OZIE-991 action prepare executions work only with HDFS filesystems (tucu)
Modified:
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
oozie/branches/branch-3.3/release-log.txt
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/FileSystemActions.java Wed Sep 12 20:35:25 2012
@@ -18,8 +18,8 @@
package org.apache.oozie.action.hadoop;
import java.io.IOException;
+import java.util.Collection;
import java.util.HashSet;
-import java.util.Set;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
@@ -30,13 +30,9 @@ import org.w3c.dom.Node;
*
*/
public class FileSystemActions {
- private static Set<String> supportedFileSystems = new HashSet<String>();
+ private static Collection<String> supportedFileSystems;
- public FileSystemActions() {
- supportedFileSystems.add("hdfs");
- }
-
- public FileSystemActions(Set<String> fileSystems) {
+ public FileSystemActions(Collection<String> fileSystems) {
supportedFileSystems = fileSystems;
}
@@ -102,10 +98,12 @@ public class FileSystemActions {
String nullScheme = "Scheme of the path " + path + " is null";
System.out.println(nullScheme);
throw new LauncherException(nullScheme);
- } else if (!supportedFileSystems.contains(scheme.toLowerCase())) {
- String unsupportedScheme = "Scheme of the provided path " + path + " is of type not supported.";
- System.out.println(unsupportedScheme);
- throw new LauncherException(unsupportedScheme);
+ } else if (supportedFileSystems.size() != 1 || !supportedFileSystems.iterator().next().equals("*")) {
+ if (!supportedFileSystems.contains(scheme.toLowerCase())) {
+ String unsupportedScheme = "Scheme of '" + path + "' is not supported.";
+ System.out.println(unsupportedScheme);
+ throw new LauncherException(unsupportedScheme);
+ }
}
} else if (scheme != null) {
String notNullScheme = "Scheme of the path " + path + " is not null as specified.";
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java Wed Sep 12 20:35:25 2012
@@ -565,7 +565,8 @@ public class JavaActionExecutor extends
prepareXML);
LauncherMapper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml));
-
+ LauncherMapper.setupSupportedFileSystems(
+ launcherJobConf, Services.get().getConf().get(HadoopAccessorService.SUPPORTED_FILESYSTEMS));
LauncherMapper.setupMaxOutputData(launcherJobConf, maxActionOutputLen);
LauncherMapper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize);
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Wed Sep 12 20:35:25 2012
@@ -55,6 +55,7 @@ import org.apache.oozie.util.XLog;
public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, Runnable {
public static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class";
+ public static final String CONF_OOZIE_ACTION_SUPPORTED_FILESYSTEMS = "oozie.launcher.action.supported.filesystems";
public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = "oozie.action.max.output.data";
@@ -148,6 +149,10 @@ public class LauncherMapper<K1, V1, K2,
launcherConf.set(CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
}
+ public static void setupSupportedFileSystems(Configuration launcherConf, String supportedFileSystems) {
+ launcherConf.set(CONF_OOZIE_ACTION_SUPPORTED_FILESYSTEMS, supportedFileSystems);
+ }
+
public static void setupMainArguments(Configuration launcherConf, String[] args) {
launcherConf.setInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
for (int i = 0; i < args.length; i++) {
@@ -642,7 +647,8 @@ public class LauncherMapper<K1, V1, K2,
String prepareXML = getJobConf().get(ACTION_PREPARE_XML);
if (prepareXML != null) {
if (!prepareXML.equals("")) {
- PrepareActionsDriver.doOperations(prepareXML);
+ PrepareActionsDriver.doOperations(
+ getJobConf().getStringCollection(CONF_OOZIE_ACTION_SUPPORTED_FILESYSTEMS), prepareXML);
} else {
System.out.println("There are no prepare actions to execute.");
}
Modified: oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java (original)
+++ oozie/branches/branch-3.3/core/src/main/java/org/apache/oozie/action/hadoop/PrepareActionsDriver.java Wed Sep 12 20:35:25 2012
@@ -20,11 +20,14 @@ package org.apache.oozie.action.hadoop;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.oozie.service.HadoopAccessorService;
import org.xml.sax.SAXException;
-import org.apache.oozie.action.hadoop.FileSystemActions;
import org.w3c.dom.Document;
import org.w3c.dom.NodeList;
+
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
@@ -41,14 +44,14 @@ public class PrepareActionsDriver {
* @param prepareXML Prepare XML block in string format
* @throws LauncherException
*/
- static void doOperations(String prepareXML) throws LauncherException {
+ static void doOperations(Collection<String> supportedFileSystems, String prepareXML) throws LauncherException {
try {
Document doc = getDocumentFromXML(prepareXML);
doc.getDocumentElement().normalize();
// Get the list of child nodes, basically, each one corresponding to a separate action
NodeList nl = doc.getDocumentElement().getChildNodes();
- FileSystemActions fsActions = new FileSystemActions();
+ FileSystemActions fsActions = new FileSystemActions(supportedFileSystems);
for (int i = 0; i < nl.getLength(); ++i) {
String commandType = "";
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestFileSystemActions.java Wed Sep 12 20:35:25 2012
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.service.Services;
import org.w3c.dom.Document;
@@ -28,6 +29,7 @@ import org.w3c.dom.Node;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
+import java.util.Arrays;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -63,7 +65,7 @@ public class TestFileSystemActions exten
Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
Node n = doc.getDocumentElement().getChildNodes().item(0);
- new FileSystemActions().execute(n);
+ new FileSystemActions(Arrays.asList("hdfs")).execute(n);
assertFalse(fs.exists(newDir));
}
@@ -84,7 +86,7 @@ public class TestFileSystemActions exten
Document doc = PrepareActionsDriver.getDocumentFromXML(prepareXML);
Node n = doc.getDocumentElement().getChildNodes().item(0);
- new FileSystemActions().execute(n);
+ new FileSystemActions(Arrays.asList("hdfs")).execute(n);
assertTrue(fs.exists(newDir));
}
@@ -92,7 +94,7 @@ public class TestFileSystemActions exten
public void testForInvalidScheme() throws Exception {
Path actionDir = getFsTestCaseDir();
// Construct a path with invalid scheme
- Path newDir = new Path("http" + actionDir.toString().substring(4) + "/delete");
+ Path newDir = new Path("file:/" + actionDir.toString().substring(5) + "/delete");
// Construct prepare XML block with the path
String prepareXML = "<prepare>" + "<delete path='" + newDir + "'/>" + "</prepare>";
// Parse the XML to get the node
@@ -100,11 +102,11 @@ public class TestFileSystemActions exten
Node n = doc.getDocumentElement().getChildNodes().item(0);
try {
- new FileSystemActions().execute(n);
+ new FileSystemActions(Arrays.asList("hdfs")).execute(n);
fail("Expected to catch an exception but did not encounter any");
} catch (LauncherException le) {
Path path = new Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
- assertEquals(le.getMessage(), "Scheme of the provided path " + path + " is of type not supported.");
+ assertEquals("Scheme of '" + path + "' is not supported.", le.getMessage());
} catch(Exception ex){
fail("Expected a LauncherException but received an Exception");
}
@@ -127,11 +129,11 @@ public class TestFileSystemActions exten
Node n = doc.getDocumentElement().getChildNodes().item(0);
try {
- new FileSystemActions().execute(n);
+ new FileSystemActions(Arrays.asList("hdfs")).execute(n);
fail("Expected to catch an exception but did not encounter any");
} catch (LauncherException le) {
Path path = new Path(n.getAttributes().getNamedItem("path").getNodeValue().trim());
- assertEquals(le.getMessage(), "Scheme of the path " + path + " is null");
+ assertEquals("Scheme of the path " + path + " is null", le.getMessage());
} catch(Exception ex) {
fail("Expected a LauncherException but received an Exception");
}
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java Wed Sep 12 20:35:25 2012
@@ -259,6 +259,7 @@ public class TestJavaActionExecutor exte
assertEquals("MAIN-CLASS", ae.getLauncherMain(conf, actionXml));
assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPTS"));
assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherMapper.getMainArguments(conf)));
+ assertNotNull(conf.get(LauncherMapper.CONF_OOZIE_ACTION_SUPPORTED_FILESYSTEMS));
assertTrue(getFileSystem().exists(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML)));
Modified: oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java (original)
+++ oozie/branches/branch-3.3/core/src/test/java/org/apache/oozie/action/hadoop/TestPrepareActionsDriver.java Wed Sep 12 20:35:25 2012
@@ -18,6 +18,7 @@
package org.apache.oozie.action.hadoop;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.service.Services;
@@ -50,7 +51,7 @@ public class TestPrepareActionsDriver ex
fs.delete(newDir, true);
}
- PrepareActionsDriver.doOperations(prepareXML);
+ PrepareActionsDriver.doOperations(Arrays.asList("hdfs"), prepareXML);
assertTrue(fs.exists(actionDir));
}
@@ -68,7 +69,7 @@ public class TestPrepareActionsDriver ex
try {
prepareXML = "prepare>" + "<mkdir path='" + newDir + "'/>" + "</prepare>";
- PrepareActionsDriver.doOperations(prepareXML);
+ PrepareActionsDriver.doOperations(Arrays.asList("hdfs"), prepareXML);
fail("Expected to catch an exception but did not encounter any");
} catch (LauncherException le) {
assertEquals(le.getCause().getClass(), org.xml.sax.SAXParseException.class);
Modified: oozie/branches/branch-3.3/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/branches/branch-3.3/release-log.txt?rev=1384113&r1=1384112&r2=1384113&view=diff
==============================================================================
--- oozie/branches/branch-3.3/release-log.txt (original)
+++ oozie/branches/branch-3.3/release-log.txt Wed Sep 12 20:35:25 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (unreleased)
+OOZIE-991 action prepare executions work only with HDFS filesystems (tucu)
OOZIE-981 Subworkflow lib not found in classpath when parent workflow lib overwrites it (mona via virag)
OOZIE-978 Bundle status doesn't transit to KILLED after a coordinator job fails submission (virag)
OOZIE-971 TestRecoveryService failing very often in pre-commit builds (mona via virag)