You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ry...@apache.org on 2014/02/01 10:11:50 UTC
git commit: OOZIE-1675 Adding absolute URI of local cluster to dist
cache not working with hadoop version 0.20.2 and before (satish via ryota)
Updated Branches:
refs/heads/master 1eaf7b9eb -> ab67228ce
OOZIE-1675 Adding absolute URI of local cluster to dist cache not working with hadoop version 0.20.2 and before (satish via ryota)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ab67228c
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ab67228c
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ab67228c
Branch: refs/heads/master
Commit: ab67228ce31afac65f668579507f6e1c17a8301e
Parents: 1eaf7b9
Author: egashira <ry...@yahoo.com>
Authored: Sat Feb 1 01:10:51 2014 -0800
Committer: egashira <ry...@yahoo.com>
Committed: Sat Feb 1 01:10:51 2014 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 14 ++++++++++-
.../action/hadoop/TestJavaActionExecutor.java | 26 +++++++++++++++++++-
release-log.txt | 1 +
3 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/ab67228c/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 5cd9eb0..68d77a8 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -426,7 +426,19 @@ public class JavaActionExecutor extends ActionExecutor {
else if (fileName.endsWith(".jar")) { // .jar files
if (!fileName.contains("#")) {
String user = conf.get("user.name");
- Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, new Path(uri.normalize()), conf);
+ Path pathToAdd;
+ // if filePath and appPath belong to same cluster, add URI path component else add absolute URI
+ if (uri.getScheme() != null && uri.getHost() != null &&
+ uri.getPort() > 0 && baseUri.getScheme() != null &&
+ baseUri.getHost() != null && baseUri.getPort() > 0 &&
+ uri.getScheme().equalsIgnoreCase(baseUri.getScheme()) &&
+ uri.getHost().equalsIgnoreCase(baseUri.getHost()) &&
+ uri.getPort() == baseUri.getPort()) {
+ pathToAdd = new Path(uri.getPath());
+ } else {
+ pathToAdd = new Path(uri.normalize());
+ }
+ Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, pathToAdd, conf);
}
else {
DistributedCache.addCacheFile(uri, conf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/ab67228c/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 88af9b5..7841076 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -1796,12 +1796,36 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertTrue(conf.get("mapred.cache.files").contains(appSoFragmentFullPath.toString()));
assertTrue(DistributedCache.getSymlink(conf));
- // test .jar without fragment
+ // test .jar without fragment where app path is on same cluster as jar path
Path appJarPath = new Path("lib/a.jar");
Path appJarFullPath = new Path(appPath, appJarPath);
conf.clear();
conf.set(WorkflowAppService.HADOOP_USER, getTestUser());
ae.addToCache(conf, appPath, appJarFullPath.toString(), false);
+ // assert that mapred.cache.files contains jar URI path
+ Path jarPath = new Path(appJarFullPath.toUri().getPath());
+ assertTrue(conf.get("mapred.cache.files").contains(jarPath.toString()));
+ // assert that dist cache classpath contains jar URI path
+ Path[] paths = DistributedCache.getFileClassPaths(conf);
+ boolean pathFound = false;
+ for (Path path : paths) {
+ if (path.equals(jarPath)) {
+ pathFound = true;
+ break;
+ }
+ }
+ assertTrue(pathFound);
+ assertTrue(DistributedCache.getSymlink(conf));
+
+ // test .jar without fragment where app path is on a different cluster than jar path
+ appJarPath = new Path("lib/a.jar");
+ appJarFullPath = new Path(appPath, appJarPath);
+ Path appDifferentClusterPath = new Path(new URI(appUri.getScheme(), null, appUri.getHost() + "x",
+ appUri.getPort(), appUri.getPath(), appUri.getQuery(), appUri.getFragment()));
+ conf.clear();
+ conf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ ae.addToCache(conf, appDifferentClusterPath, appJarFullPath.toString(), false);
+ // assert that mapred.cache.files contains absolute jar URI
assertTrue(conf.get("mapred.cache.files").contains(appJarFullPath.toString()));
assertTrue(DistributedCache.getSymlink(conf));
http://git-wip-us.apache.org/repos/asf/oozie/blob/ab67228c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 2b70dfc..f424dd3 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1675 Adding absolute URI of local cluster to dist cache not working with hadoop version 0.20.2 and before (satish via ryota)
OOZIE-1683 UserGroupInformationService should close any filesystems opened by it (rkanter)
OOZIE-1646 HBase Table Copy between two HBase servers doesn't work with Kerberos (rkanter)
OOZIE-1609 HA support for share lib. (puru via rkanter)