You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/08/21 23:31:32 UTC
svn commit: r1375804 - in /incubator/oozie/trunk: ./
client/src/main/java/org/apache/oozie/client/
core/src/main/java/org/apache/oozie/
core/src/main/java/org/apache/oozie/action/hadoop/
core/src/main/java/org/apache/oozie/service/ core/src/main/resour...
Author: tucu
Date: Tue Aug 21 21:31:31 2012
New Revision: 1375804
URL: http://svn.apache.org/viewvc?rev=1375804&view=rev
Log:
OOZIE-477 Adding configurable filesystem support instead of hardcoded hdfs (mayank, mona via tucu)
Modified:
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
incubator/oozie/trunk/core/src/main/resources/oozie-default.xml
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java (original)
+++ incubator/oozie/trunk/client/src/main/java/org/apache/oozie/client/XOozieClient.java Tue Aug 21 21:31:31 2012
@@ -113,8 +113,17 @@ public class XOozieClient extends OozieC
if (libPath == null) {
throw new RuntimeException("libpath is not specified in conf");
}
- if (!libPath.startsWith("hdfs://")) {
- String newLibPath = NN + libPath;
+ if (!libPath.contains(":/")) {
+ String newLibPath;
+ if (libPath.startsWith("/")) {
+ if(NN.endsWith("/")) {
+ newLibPath = NN + libPath.substring(1);
+ } else {
+ newLibPath = NN + libPath;
+ }
+ } else {
+ throw new RuntimeException("libpath should be absolute");
+ }
conf.setProperty(LIBPATH, newLibPath);
}
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/ErrorCode.java Tue Aug 21 21:31:31 2012
@@ -172,6 +172,7 @@ public enum ErrorCode {
E0901(XLog.OPS, "Namenode [{0}] not allowed, not in Oozie's whitelist"),
E0902(XLog.OPS, "Exception occured: [{0}]"),
E0903(XLog.OPS, "Invalid JobConf, it has not been created by HadoopAccessorService"),
+ E0904(XLog.STD, "Scheme [{0}] not supported in uri [{1}]"),
E1001(XLog.STD, "Could not read the coordinator job definition, {0}"),
E1002(XLog.STD, "Invalid coordinator application URI [{0}], {1}"),
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java Tue Aug 21 21:31:31 2012
@@ -54,30 +54,32 @@ public class FsActionExecutor extends Ac
}
void validatePath(Path path, boolean withScheme) throws ActionExecutorException {
- String scheme = path.toUri().getScheme();
- if (withScheme) {
- if (scheme == null) {
- throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS001",
- "Missing scheme in path [{0}]", path);
+ try {
+ String scheme = path.toUri().getScheme();
+ if (withScheme) {
+ if (scheme == null) {
+ throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS001",
+ "Missing scheme in path [{0}]", path);
+ }
+ else {
+ Services.get().get(HadoopAccessorService.class).checkSupportedFilesystem(path.toUri());
+ }
}
else {
- if (!scheme.equals("hdfs")) {
+ if (scheme != null) {
throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS002",
- "Scheme [{0}] not supported in path [{1}]", scheme, path);
+ "Scheme [{0}] not allowed in path [{1}]", scheme, path);
}
}
}
- else {
- if (scheme != null) {
- throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "FS003",
- "Scheme [{0}] not allowed in path [{1}]", scheme, path);
- }
+ catch (HadoopAccessorException hex) {
+ throw convertException(hex);
}
}
-
+
Path resolveToFullPath(Path nameNode, Path path, boolean withScheme) throws ActionExecutorException {
Path fullPath;
-
+
// If no nameNode is given, validate the path as-is and return it as-is
if (nameNode == null) {
validatePath(path, withScheme);
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java Tue Aug 21 21:31:31 2012
@@ -265,6 +265,9 @@ public class JavaActionExecutor extends
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
XConfiguration actionDefaults = has.createActionDefaultConf(actionConf.get(HADOOP_JOB_TRACKER), getType());
XConfiguration.injectDefaults(actionDefaults, actionConf);
+
+ has.checkSupportedFilesystem(appPath.toUri());
+
parseJobXmlAndConfiguration(context, actionXml, appPath, actionConf);
return actionConf;
}
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java Tue Aug 21 21:31:31 2012
@@ -30,6 +30,7 @@ import java.io.StringWriter;
import java.io.Writer;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.net.URI;
import java.security.Permission;
import java.text.MessageFormat;
import java.util.Properties;
@@ -87,9 +88,9 @@ public class LauncherMapper<K1, V1, K2,
private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException {
try {
- FileSystem fs = FileSystem.get(launcherConf);
String jobId = launcherConf.get("mapred.job.id");
Path path = new Path(actionDir, recoveryId);
+ FileSystem fs = FileSystem.get(path.toUri(), launcherConf);
if (!fs.exists(path)) {
try {
Writer writer = new OutputStreamWriter(fs.create(path));
@@ -130,7 +131,6 @@ public class LauncherMapper<K1, V1, K2,
throws HadoopAccessorException, IOException {
String jobId = null;
Path recoveryFile = new Path(actionDir, recoveryId);
- //FileSystem fs = FileSystem.get(launcherConf);
FileSystem fs = Services.get().get(HadoopAccessorService.class)
.createFileSystem(launcherConf.get("user.name"),recoveryFile.toUri(), launcherConf);
@@ -498,9 +498,10 @@ public class LauncherMapper<K1, V1, K2,
}
if (errorMessage == null) {
File outputData = new File(System.getProperty("oozie.action.output.properties"));
- FileSystem fs = FileSystem.get(getJobConf());
+ FileSystem fs = null;
if (outputData.exists()) {
-
+ URI actionDirUri = new Path(actionDir, ACTION_OUTPUT_PROPS).toUri();
+ fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(outputData.toString()), new Path(actionDir,
ACTION_OUTPUT_PROPS));
reporter.incrCounter(COUNTER_GROUP, COUNTER_OUTPUT_DATA, 1);
@@ -530,7 +531,8 @@ public class LauncherMapper<K1, V1, K2,
if (props.getProperty("id") == null) {
throw new IllegalStateException("ID swap file does not have [id] property");
}
- fs = FileSystem.get(getJobConf());
+ URI actionDirUri = new Path(actionDir, ACTION_NEW_ID_PROPS).toUri();
+ fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(newId.toString()), new Path(actionDir, ACTION_NEW_ID_PROPS));
reporter.incrCounter(COUNTER_GROUP, COUNTER_DO_ID_SWAP, 1);
@@ -585,7 +587,7 @@ public class LauncherMapper<K1, V1, K2,
return jobConf;
}
- private void handleActionStatsData(FileSystem fs, Reporter reporter) throws IOException, LauncherException{
+ private void handleActionStatsData(FileSystem fs, Reporter reporter) throws IOException, LauncherException {
File actionStatsData = new File(System.getProperty(EXTERNAL_ACTION_STATS));
// If stats are stored by the action, then stats file should exist
if (actionStatsData.exists()) {
@@ -599,6 +601,8 @@ public class LauncherMapper<K1, V1, K2,
failLauncher(0, msg, null);
}
// copy the stats file to hdfs path which can be accessed by Oozie server
+ URI actionDirUri = new Path(actionDir, ACTION_STATS_PROPS).toUri();
+ fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(actionStatsData.toString()), new Path(actionDir,
ACTION_STATS_PROPS));
}
@@ -609,15 +613,19 @@ public class LauncherMapper<K1, V1, K2,
// if external ChildIDs are stored by the action, then the file should exist
if (externalChildIDs.exists()) {
// copy the externalChildIDs file to hdfs path which can be accessed by Oozie server
+ URI actionDirUri = new Path(actionDir, ACTION_EXTERNAL_CHILD_IDS_PROPS).toUri();
+ fs = FileSystem.get(actionDirUri, getJobConf());
fs.copyFromLocalFile(new Path(externalChildIDs.toString()), new Path(actionDir,
ACTION_EXTERNAL_CHILD_IDS_PROPS));
}
}
- private void setupMainConfiguration() throws IOException {
- FileSystem fs = FileSystem.get(getJobConf());
- fs.copyToLocalFile(new Path(getJobConf().get(OOZIE_ACTION_DIR_PATH), ACTION_CONF_XML), new Path(new File(
- ACTION_CONF_XML).getAbsolutePath()));
+ private void setupMainConfiguration() throws IOException, HadoopAccessorException {
+ Path pathNew = new Path(new Path(actionDir, ACTION_CONF_XML),
+ new Path(new File(ACTION_CONF_XML).getAbsolutePath()));
+ FileSystem fs = FileSystem.get(pathNew.toUri(), getJobConf());
+ fs.copyToLocalFile(new Path(actionDir, ACTION_CONF_XML),
+ new Path(new File(ACTION_CONF_XML).getAbsolutePath()));
System.setProperty("oozie.launcher.job.id", getJobConf().get("mapred.job.id"));
System.setProperty("oozie.job.id", getJobConf().get(OOZIE_JOB_ID));
@@ -688,7 +696,7 @@ public class LauncherMapper<K1, V1, K2,
pw.close();
errorProps.setProperty("exception.stacktrace", sw.toString());
}
- FileSystem fs = FileSystem.get(getJobConf());
+ FileSystem fs = FileSystem.get((new Path(actionDir, ACTION_ERROR_PROPS)).toUri(), getJobConf());
OutputStream os = fs.create(new Path(actionDir, ACTION_ERROR_PROPS));
errorProps.store(os, "");
os.close();
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java Tue Aug 21 21:31:31 2012
@@ -73,6 +73,12 @@ public class HadoopAccessorService imple
private ConcurrentMap<String, UserGroupInformation> userUgiMap;
+ /**
+ * Supported filesystem schemes for namespace federation
+ */
+ public static final String SUPPORTED_FILESYSTEMS = CONF_PREFIX + "supported.filesystems";
+ private Set<String> supportedSchemes;
+
public void init(Services services) throws ServiceException {
init(services.getConf());
}
@@ -115,6 +121,23 @@ public class HadoopAccessorService imple
loadHadoopConfigs(conf);
preLoadActionConfigs(conf);
+
+ supportedSchemes = new HashSet<String>();
+ String[] schemesFromConf = conf.getStrings(SUPPORTED_FILESYSTEMS, new String[]{"hdfs"});
+ if(schemesFromConf != null) {
+ for (String scheme: schemesFromConf) {
+ scheme = scheme.trim();
+ // If user gives "*", supportedSchemes will be empty, so that checking is not done i.e. all schemes allowed
+ if(scheme.equals("*")) {
+ if(schemesFromConf.length > 1) {
+ throw new ServiceException(ErrorCode.E0100, getClass().getName(),
+ SUPPORTED_FILESYSTEMS + " should contain either only wildcard or explicit list, not both");
+ }
+ } else {
+ supportedSchemes.add(scheme);
+ }
+ }
+ }
}
private void kerberosInit(Configuration serviceConf) throws ServiceException {
@@ -364,6 +387,9 @@ public class HadoopAccessorService imple
if (!conf.getBoolean(OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, false)) {
throw new HadoopAccessorException(ErrorCode.E0903);
}
+
+ checkSupportedFilesystem(uri);
+
String nameNode = uri.getAuthority();
if (nameNode == null) {
nameNode = conf.get("fs.default.name");
@@ -445,4 +471,19 @@ public class HadoopAccessorService imple
}
+ /**
+ * checks configuration parameter if filesystem scheme is among the list of supported ones
+ * this makes system robust to filesystems other than HDFS also
+ */
+
+ public void checkSupportedFilesystem(URI uri) throws HadoopAccessorException {
+ String uriScheme = uri.getScheme();
+ if(!supportedSchemes.isEmpty()) {
+ XLog.getLog(this.getClass()).debug("Checking if filesystem " + uriScheme + " is supported");
+ if (!supportedSchemes.contains(uriScheme)) {
+ throw new HadoopAccessorException(ErrorCode.E0904, uriScheme, uri.toString());
+ }
+ }
+ }
+
}
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java Tue Aug 21 21:31:31 2012
@@ -37,10 +37,8 @@ import java.io.Reader;
import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -168,7 +166,7 @@ public abstract class WorkflowAppService
FileSystem fs = has.createFileSystem(user, uri, conf);
- Path appPath = new Path(uri.getPath());
+ Path appPath = new Path(uri);
XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(OozieClient.LIBPATH));
XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath);
@@ -259,7 +257,7 @@ public abstract class WorkflowAppService
FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
for (FileStatus file : files) {
- libPaths.add(file.getPath().toUri().getPath().trim());
+ libPaths.add(file.getPath().toUri().toString());
}
}
else {
Modified: incubator/oozie/trunk/core/src/main/resources/oozie-default.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/resources/oozie-default.xml?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/resources/oozie-default.xml (original)
+++ incubator/oozie/trunk/core/src/main/resources/oozie-default.xml Tue Aug 21 21:31:31 2012
@@ -1639,4 +1639,13 @@
Set it true only if backward compatibility for action/job info is required.
</description>
</property>
+
+ <property>
+ <name>oozie.service.HadoopAccessorService.supported.filesystems</name>
+ <value>hdfs</value>
+ <description>
+ Enlist the different filesystems supported for federation. If wildcard "*" is specified,
+ then ALL file schemes will be allowed.
+ </description>
+ </property>
</configuration>
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java Tue Aug 21 21:31:31 2012
@@ -17,15 +17,18 @@
*/
package org.apache.oozie.action.hadoop;
-import java.io.OutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.oozie.ErrorCode;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.ServiceException;
+import org.apache.oozie.service.Services;
import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XmlUtils;
@@ -52,7 +55,6 @@ public class TestFsActionExecutor extend
XConfiguration protoConf = new XConfiguration();
protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-
WorkflowJobBean wf = createBaseWorkflow(protoConf, "fs-action");
WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
action.setType(ae.getType());
@@ -71,7 +73,7 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS003", ex.getErrorCode());
+ assertEquals("FS002", ex.getErrorCode());
}
try {
@@ -79,30 +81,55 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS001", ex.getErrorCode());
+ assertEquals("FS001", ex.getErrorCode());
+ }
+
+ // testing schemes supported
+ setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "hdfs,viewfs");
+ new Services().init();
+ try {
+ ae.validatePath(new Path("viewfs://bla"), true);
+ }
+ catch (ActionExecutorException ex) {
+ fail("viewfs is a supported scheme. This should not throw exception");
}
try {
ae.validatePath(new Path("file://bla"), true);
- fail();
+
+ fail("file is not a supported scheme. This should throw exception");
}
catch (ActionExecutorException ex) {
- assertEquals("FS002", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("E0904"));
}
}
-
+
+ public void testFileSchemeWildcard() throws Exception {
+ FsActionExecutor ae = new FsActionExecutor();
+ setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "*");
+ new Services().init();
+
+ try {
+ ae.validatePath(new Path("anyfs://bla"), true);
+ }
+ catch (ActionExecutorException ex) {
+ fail("Wildcard indicates ALL schemes will be allowed. This should pass");
+ }
+ }
+
+
public void testResolveToFullPath() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
-
+
assertEquals(new Path("hdfs://x/bla"), ae.resolveToFullPath(null, new Path("hdfs://x/bla"), true));
assertEquals(new Path("bla"), ae.resolveToFullPath(null, new Path("bla"), false));
-
+
assertEquals(new Path("hdfs://x/bla"), ae.resolveToFullPath(new Path("hdfs://x"), new Path("/bla"), true));
-
+
assertEquals(new Path("hdfs://x/bla"), ae.resolveToFullPath(new Path("hdfs://x/ha"), new Path("/bla"), true));
-
+
assertEquals(new Path("hdfs://x/bla"), ae.resolveToFullPath(new Path("hdfs://z"), new Path("hdfs://x/bla"), true));
-
+
assertEquals(new Path("hdfs://x/bla"), ae.resolveToFullPath(new Path("hdfs://x"), new Path("hdfs://x/bla"), true));
try {
@@ -110,7 +137,7 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS003", ex.getErrorCode());
+ assertEquals("FS002", ex.getErrorCode());
}
try {
@@ -121,20 +148,22 @@ public class TestFsActionExecutor extend
assertEquals("FS001", ex.getErrorCode());
}
+ setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, null);
+ new Services().init();
try {
ae.resolveToFullPath(null, new Path("file://bla"), true);
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS002", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("E0904"));
}
-
+
try {
ae.resolveToFullPath(new Path("hdfs://z"), new Path("hdfs://x/bla"), false);
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS003", ex.getErrorCode());
+ assertEquals("FS002", ex.getErrorCode());
}
try {
@@ -150,10 +179,10 @@ public class TestFsActionExecutor extend
fail();
}
catch (ActionExecutorException ex) {
- assertEquals("FS002", ex.getErrorCode());
+ assertTrue(ex.getMessage().contains("E0904"));
}
}
-
+
public void testvalidateSameNN() throws Exception {
FsActionExecutor ae = new FsActionExecutor();
ae.validateSameNN(new Path("hdfs://x/bla"), new Path("hdfs://x/foo"));
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java Tue Aug 21 21:31:31 2012
@@ -1000,6 +1000,28 @@ public class TestJavaActionExecutor exte
assertTrue(cacheFilesStr.contains(jar3Path.toString()));
}
+ public void testFilesystemScheme() throws Exception {
+ try {
+ String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName()
+ + "</main-class>" + "</java>";
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+ Context context = createContext(actionXml);
+ Path appPath = new Path("localfs://namenode:port/mydir");
+ JavaActionExecutor ae = new JavaActionExecutor();
+ JobConf conf = ae.createBaseHadoopConf(context, eActionXml);
+ setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "hdfs,viewfs");
+ new Services().init();
+ ae.setupActionConf(conf, context, eActionXml, appPath);
+
+ fail("Supposed to throw exception due to unsupported fs scheme - localfs");
+ }
+ catch (ActionExecutorException ae) {
+ assertTrue(ae.getMessage().contains("E0904"));
+ assertTrue(ae.getMessage().contains("Scheme [localfs] not supported"));
+ }
+ }
+
public void testACLDefaults_launcherACLsSetToDefault() throws Exception {
// CASE: launcher specific ACLs not configured - set defaults
String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowAppService.java Tue Aug 21 21:31:31 2012
@@ -328,8 +328,8 @@ public class TestLiteWorkflowAppService
assertEquals(2, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
- String ref1 = getTestCaseDir() + "/lib/reduceutil.so";
- String ref2 = getTestCaseDir() + "/lib/maputil.jar";
+ String ref1 = "file://" + getTestCaseDir() + "/lib/reduceutil.so";
+ String ref2 = "file://" + getTestCaseDir() + "/lib/maputil.jar";
Assert.assertTrue(f1.equals(ref1) || f1.equals(ref2));
Assert.assertTrue(f2.equals(ref1) || f2.equals(ref2));
Assert.assertTrue(!f1.equals(f2));
@@ -373,9 +373,9 @@ public class TestLiteWorkflowAppService
found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1]);
found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[2]);
List<String> expected = new ArrayList<String>();
- expected.add(getTestCaseDir() + "/lib/reduceutil.so");
- expected.add(getTestCaseDir() + "/lib/maputil.jar");
- expected.add(getTestCaseDir() + "/libx/maputilx.jar");
+ expected.add("file://" + getTestCaseDir() + "/lib/reduceutil.so");
+ expected.add("file://" + getTestCaseDir() + "/lib/maputil.jar");
+ expected.add("file://" + getTestCaseDir() + "/libx/maputilx.jar");
Collections.sort(found);
Collections.sort(expected);
assertEquals(expected, found);
@@ -435,12 +435,12 @@ public class TestLiteWorkflowAppService
found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[4]);
found.add(protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[5]);
List<String> expected = new ArrayList<String>();
- expected.add(getTestCaseDir() + "/lib/reduceutil.so");
- expected.add(getTestCaseDir() + "/lib/maputil.jar");
- expected.add(getTestCaseDir() + "/libx/maputil_x.jar");
- expected.add(getTestCaseDir() + "/liby/maputil_y1.jar");
- expected.add(getTestCaseDir() + "/liby/maputil_y2.jar");
- expected.add(getTestCaseDir() + "/libz/maputil_z.jar");
+ expected.add("file://" + getTestCaseDir() + "/lib/reduceutil.so");
+ expected.add("file://" + getTestCaseDir() + "/lib/maputil.jar");
+ expected.add("file://" + getTestCaseDir() + "/libx/maputil_x.jar");
+ expected.add("file://" + getTestCaseDir() + "/liby/maputil_y1.jar");
+ expected.add("file://" + getTestCaseDir() + "/liby/maputil_y2.jar");
+ expected.add("file://" + getTestCaseDir() + "/libz/maputil_z.jar");
Collections.sort(found);
Collections.sort(expected);
assertEquals(expected, found);
@@ -479,8 +479,8 @@ public class TestLiteWorkflowAppService
assertEquals(2, protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST).length);
String f1 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[0];
String f2 = protoConf.getStrings(WorkflowAppService.APP_LIB_PATH_LIST)[1];
- String ref1 = getTestCaseDir() + "/lib/childdependency1.jar";
- String ref2 = getTestCaseDir() + "/lib/childdependency2.so";
+ String ref1 = "file://" + getTestCaseDir() + "/lib/childdependency1.jar";
+ String ref2 = "file://" + getTestCaseDir() + "/lib/childdependency2.so";
List<String> expected = new ArrayList<String>();
expected.add(ref1);
expected.add(ref2);
Modified: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java (original)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/test/XTestCase.java Tue Aug 21 21:31:31 2012
@@ -54,6 +54,7 @@ import org.apache.oozie.SLAEventBean;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.HadoopAccessorService;
import org.apache.oozie.service.Services;
import org.apache.oozie.store.CoordinatorStore;
import org.apache.oozie.store.StoreException;
@@ -285,6 +286,8 @@ public abstract class XTestCase extends
}
setSystemProperty(ConfigurationService.OOZIE_DATA_DIR, testCaseDir);
+ setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS,"*");
+
if (mrCluster != null) {
OutputStream os = new FileOutputStream(new File(hadoopConfDir, "core-site.xml"));
Configuration conf = mrCluster.createJobConf();
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1375804&r1=1375803&r2=1375804&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Tue Aug 21 21:31:31 2012
@@ -1,5 +1,6 @@
-- Oozie 3.3.0 release (trunk - unreleased)
+OOZIE-477 Adding configurable filesystem support instead of hardcoded "hdfs" (mayank, mona via tucu)
OOZIE-906 Show runtime job DAG visually in Oozie console/dashboard (vaidya via virag)
OOZIE-958 typo in error message of E0736 (egashira via virag)
OOZIE-923 Improve error message when a user tries to start a coordinator job (sms via virag)