You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2016/08/11 19:44:50 UTC
oozie git commit: OOZIE-2512 ShareLibservice returns incorrect path
for jar
Repository: oozie
Updated Branches:
refs/heads/master bec42a030 -> 16fa65946
OOZIE-2512 ShareLibservice returns incorrect path for jar
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/16fa6594
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/16fa6594
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/16fa6594
Branch: refs/heads/master
Commit: 16fa6594689d86dc4f4a62124f2b1140f0b9368d
Parents: bec42a0
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Thu Aug 11 12:44:44 2016 -0700
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Thu Aug 11 12:44:44 2016 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 4 +--
.../apache/oozie/service/ShareLibService.java | 10 +++----
.../oozie/service/TestShareLibService.java | 29 +++++++++++++++++++-
release-log.txt | 1 +
4 files changed, 35 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 9e1682c..e546e77 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -683,12 +683,10 @@ public class JavaActionExecutor extends ActionExecutor {
if (listOfPaths != null && !listOfPaths.isEmpty()) {
for (Path actionLibPath : listOfPaths) {
String fragmentName = new URI(actionLibPath.toString()).getFragment();
- Path pathWithFragment = fragmentName == null ? actionLibPath : new Path(new URI(
- actionLibPath.toString()).getPath());
String fileName = fragmentName == null ? actionLibPath.getName() : fragmentName;
if (confSet.contains(fileName)) {
Configuration jobXmlConf = shareLibService.getShareLibConf(actionShareLibName,
- pathWithFragment);
+ actionLibPath);
if (jobXmlConf != null) {
checkForDisallowedProps(jobXmlConf, actionLibPath.getName());
XConfiguration.injectDefaults(jobXmlConf, conf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 66fd864..b79bd37 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -289,7 +289,7 @@ public class ShareLibService implements Service, Instrumentable {
Path filePath = new Path(new URI(rootDir.toString()).getPath());
Path qualifiedRootDirPath = fs.makeQualified(rootDir);
if (isFilePartOfConfList(rootDir)) {
- cachePropertyFile(filePath, shareLibKey, shareLibConfigMap);
+ cachePropertyFile(qualifiedRootDirPath, filePath, shareLibKey, shareLibConfigMap);
}
listOfPaths.add(qualifiedRootDirPath);
return;
@@ -307,7 +307,7 @@ public class ShareLibService implements Service, Instrumentable {
}
else {
if (isFilePartOfConfList(file.getPath())) {
- cachePropertyFile(file.getPath(), shareLibKey, shareLibConfigMap);
+ cachePropertyFile(file.getPath(), file.getPath(), shareLibKey, shareLibConfigMap);
}
listOfPaths.add(file.getPath());
}
@@ -642,7 +642,7 @@ public class ShareLibService implements Service, Instrumentable {
Path path = new Path(dfsPath);
getPathRecursively(fs, new Path(dfsPath), listOfPaths, shareLibKey, shareLibConfigMap);
if (HadoopShims.isSymlinkSupported() && fileSystem.isSymlink(path)) {
- symlinkMappingforAction.put(path, fileSystem.getSymLinkTarget(path));
+ symlinkMappingforAction.put(fs.makeQualified(path), fileSystem.getSymLinkTarget(path));
}
}
if (HadoopShims.isSymlinkSupported()) {
@@ -834,7 +834,7 @@ public class ShareLibService implements Service, Instrumentable {
* @throws IOException Signals that an I/O exception has occurred.
* @throws JDOMException
*/
- private void cachePropertyFile(Path hdfsPath, String shareLibKey,
+ private void cachePropertyFile(Path qualifiedHdfsPath, Path hdfsPath, String shareLibKey,
Map<String, Map<Path, Configuration>> shareLibConfigMap) throws IOException, JDOMException {
Map<Path, Configuration> confMap = shareLibConfigMap.get(shareLibKey);
if (confMap == null) {
@@ -842,7 +842,7 @@ public class ShareLibService implements Service, Instrumentable {
shareLibConfigMap.put(shareLibKey, confMap);
}
Configuration xmlConf = new XConfiguration(fs.open(hdfsPath));
- confMap.put(hdfsPath, xmlConf);
+ confMap.put(qualifiedHdfsPath, xmlConf);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index 7de7d78..2d31f9a 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -902,7 +902,34 @@ public class TestShareLibService extends XFsTestCase {
actionConf = ae.createBaseHadoopConf(context, eActionXml);
prop.put("oozie.hive_conf", "hdfs:///user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
- jobConf.set("oozie.action.sharelib.for.hive", "hive_conf,linkFile");
+ jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+ assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
+ cacheFiles = DistributedCache.getCacheFiles(actionConf);
+ cacheFilesStr = Arrays.toString(cacheFiles);
+ assertFalse(cacheFilesStr.contains("hive-site.xml"));
+
+ // Test hive-site.xml property in jobconf with non hdfs path
+ prop = new Properties();
+ jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ actionConf = ae.createBaseHadoopConf(context, eActionXml);
+ prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml");
+ setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
+ jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+ assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
+ cacheFiles = DistributedCache.getCacheFiles(actionConf);
+ cacheFilesStr = Arrays.toString(cacheFiles);
+ assertFalse(cacheFilesStr.contains("hive-site.xml"));
+
+ // Test hive-site.xml property in jobconf with non hdfs path with
+ // link name
+ prop = new Properties();
+ jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ actionConf = ae.createBaseHadoopConf(context, eActionXml);
+ prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
+ setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
+ jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
cacheFiles = DistributedCache.getCacheFiles(actionConf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/16fa6594/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 1531194..86a68c5 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2512 ShareLibservice returns incorrect path for jar (satishsaley via puru)
OOZIE-2508 Documentation change for Coord action rerun [OOZIE-1735] (satishsaley via puru)
OOZIE-2628 Fix Workflow EL function return types in Documentation (abhishekbafna via jaydeepvishwakarma)
OOZIE-2623 Oozie should use a dummy OutputFormat (satishsaley via rohini)