You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2016/06/03 21:39:24 UTC
oozie git commit: amend-OOZIE-2512 ShareLibservice returns incorrect
path for jar
Repository: oozie
Updated Branches:
refs/heads/master c2f1edcae -> a7cc94225
amend-OOZIE-2512 ShareLibservice returns incorrect path for jar
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/a7cc9422
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/a7cc9422
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/a7cc9422
Branch: refs/heads/master
Commit: a7cc94225ed9dad7b30c530824a0274b1b334d37
Parents: c2f1edc
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Fri Jun 3 14:39:18 2016 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Fri Jun 3 14:39:18 2016 -0700
----------------------------------------------------------------------
.../apache/oozie/service/ShareLibService.java | 5 ++--
.../oozie/service/TestShareLibService.java | 31 ++++++++++++++++----
2 files changed, 28 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/a7cc9422/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index ccf9549..66fd864 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -287,12 +287,11 @@ public class ShareLibService implements Service, Instrumentable {
try {
if (fs.isFile(new Path(new URI(rootDir.toString()).getPath()))) {
Path filePath = new Path(new URI(rootDir.toString()).getPath());
- Path hdfsFilePath = new Path(
- fs.getFileStatus(filePath).getPath().getParent() + "/" + rootDir.getName());
+ Path qualifiedRootDirPath = fs.makeQualified(rootDir);
if (isFilePartOfConfList(rootDir)) {
cachePropertyFile(filePath, shareLibKey, shareLibConfigMap);
}
- listOfPaths.add(hdfsFilePath);
+ listOfPaths.add(qualifiedRootDirPath);
return;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/a7cc9422/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index b245d35..7de7d78 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -495,7 +495,11 @@ public class TestShareLibService extends XFsTestCase {
assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
assertTrue(shareLibService.getShareLibJars("directjar").get(0).getName().endsWith("direct.jar"));
- assertTrue(shareLibService.getShareLibJars("linkFile").get(0).getName().endsWith("#renamedLinkFile.xml"));
+ // Skipping for hadoop - 1.x because symlink is not supported
+ if (HadoopShims.isSymlinkSupported()) {
+ assertTrue(
+ shareLibService.getShareLibJars("linkFile").get(0).getName().endsWith("targetOfLinkFile.xml"));
+ }
List<Path> listOfPaths = shareLibService.getShareLibJars("directjar");
for (Path p : listOfPaths) {
assertTrue(p.toString().startsWith("hdfs"));
@@ -759,6 +763,7 @@ public class TestShareLibService extends XFsTestCase {
try {
String testPath = "shareLibPath/";
+ String symlink = "/user/test/linkFile.xml";
Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
Path somethingNew = new Path(testPath + Path.SEPARATOR + "something_new");
@@ -773,16 +778,18 @@ public class TestShareLibService extends XFsTestCase {
createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
createFile(somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR + "somethingNew.jar");
String directJarPath = directJarDir.toString() + Path.SEPARATOR + "direct.jar";
- String linkFilePath = linkDir.toString() + Path.SEPARATOR + "linkFile.xml";
+ String symlinkTarget = linkDir.toString() + Path.SEPARATOR + "targetOfLinkFile.xml";
createFile(directJarPath);
- createFile(linkFilePath);
+ createFile(symlinkTarget);
+ HadoopShims fsShim = new HadoopShims(fs);
+ fsShim.createSymlink(new Path(symlinkTarget), new Path(symlink), true);
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString());
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new", "/user/test/" + somethingNew.toString());
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".directjar",
"/user/test/" + directJarPath.toString());
- prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".linkFile",
- "/user/test/" + linkFilePath.toString() + "#renamedLinkFile.xml");
+ prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".linkFile", symlink + "#targetOfLinkFile.xml");
+
createTestShareLibMetaFile(fs, prop);
}
@@ -888,6 +895,20 @@ public class TestShareLibService extends XFsTestCase {
ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
+ // Test hive-site.xml property in jobconf with linkname
+ // and with hdfs path
+ prop = new Properties();
+ jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ actionConf = ae.createBaseHadoopConf(context, eActionXml);
+ prop.put("oozie.hive_conf", "hdfs:///user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
+ setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
+ jobConf.set("oozie.action.sharelib.for.hive", "hive_conf,linkFile");
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+ assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
+ cacheFiles = DistributedCache.getCacheFiles(actionConf);
+ cacheFilesStr = Arrays.toString(cacheFiles);
+ assertFalse(cacheFilesStr.contains("hive-site.xml"));
+
}
finally {
getFileSystem().delete(new Path(sharelibPath), true);