You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2014/11/10 23:27:32 UTC
oozie git commit: OOZIE-2045 Symlink support for sharelib
Repository: oozie
Updated Branches:
refs/heads/master a40535869 -> b5d409e63
OOZIE-2045 Symlink support for sharelib
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b5d409e6
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b5d409e6
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b5d409e6
Branch: refs/heads/master
Commit: b5d409e635dccef2dcf8f1137e754deb20f31461
Parents: a405358
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Mon Nov 10 14:27:11 2014 -0800
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Mon Nov 10 14:27:11 2014 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 3 +-
.../apache/oozie/service/ShareLibService.java | 155 ++++++++++++++-----
.../oozie/service/TestShareLibService.java | 132 ++++++++++++----
.../apache/oozie/hadoop/utils/HadoopShims.java | 47 ++++++
.../apache/oozie/hadoop/utils/HadoopShims.java | 48 ++++++
.../apache/oozie/hadoop/utils/HadoopShims.java | 48 ++++++
.../apache/oozie/hadoop/utils/HadoopShims.java | 48 ++++++
release-log.txt | 1 +
8 files changed, 419 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 687a948..fd1a98b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -66,7 +66,6 @@ import org.apache.oozie.service.Services;
import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
-import org.apache.oozie.servlet.CallbackServlet;
import org.apache.oozie.util.ELEvaluator;
import org.apache.oozie.util.JobUtils;
import org.apache.oozie.util.LogUtils;
@@ -80,6 +79,7 @@ import org.jdom.JDOMException;
import org.jdom.Namespace;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.oozie.hadoop.utils.HadoopShims;
public class JavaActionExecutor extends ActionExecutor {
@@ -138,6 +138,7 @@ public class JavaActionExecutor extends ActionExecutor {
classes.add(LauncherMapper.class);
classes.add(OozieLauncherInputFormat.class);
classes.add(LauncherMainHadoopUtils.class);
+ classes.add(HadoopShims.class);
classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
return classes;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 5414e6b..b408549 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -21,8 +21,10 @@ package org.apache.oozie.service;
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
+import java.text.MessageFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -38,6 +40,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;
+import java.util.Map.Entry;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileStatus;
@@ -48,6 +51,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.client.rest.JsonUtils;
+import org.apache.oozie.hadoop.utils.HadoopShims;
import org.apache.oozie.util.Instrumentable;
import org.apache.oozie.util.Instrumentation;
import org.apache.oozie.util.XLog;
@@ -82,6 +86,9 @@ public class ShareLibService implements Service, Instrumentable {
private Map<String, List<Path>> launcherLibMap = new HashMap<String, List<Path>>();
+ //symlink mapping. Oozie keeps on checking symlink path and if changes, Oozie reloads the sharelib
+ private Map<String, Map<Path, Path>> symlinkMapping = new HashMap<String, Map<Path, Path>>();
+
private static XLog LOG = XLog.getLog(ShareLibService.class);
private String sharelibMappingFile;
@@ -113,7 +120,8 @@ public class ShareLibService implements Service, Instrumentable {
fs = FileSystem.get(has.createJobConf(uri.getAuthority()));
updateLauncherLib();
updateShareLib();
- } catch(Throwable e) {
+ }
+ catch (Throwable e) {
if (failOnfailure) {
LOG.error("Sharelib initialization fails", e);
throw new ServiceException(ErrorCode.E0104, getClass().getName(), "Sharelib initialization fails. ", e);
@@ -132,7 +140,7 @@ public class ShareLibService implements Service, Instrumentable {
public void run() {
System.out.flush();
try {
- //Only one server should purge sharelib
+ // Only one server should purge sharelib
if (Services.get().get(JobsConcurrencyService.class).isLeader()) {
final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime();
purgeLibs(fs, LAUNCHER_PREFIX, current);
@@ -228,7 +236,7 @@ public class ShareLibService implements Service, Instrumentable {
* @param classes the classes
* @param fs the FileSystem
* @param executorDir is Path
- * @param type is actionKey
+ * @param type is sharelib key
* @throws IOException Signals that an I/O exception has occurred.
*/
private void copyJarContainingClasses(List<Class> classes, FileSystem fs, Path executorDir, String type)
@@ -270,6 +278,15 @@ public class ShareLibService implements Service, Instrumentable {
return;
}
+ try {
+ if(fs.isFile(new Path(new URI(rootDir.toString()).getPath()))){
+ listOfPaths.add(rootDir);
+ return;
+ }
+ }
+ catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
FileStatus[] status = fs.listStatus(rootDir);
if (status == null) {
LOG.info("Shared lib " + rootDir + " doesn't exist, not adding to cache");
@@ -290,14 +307,18 @@ public class ShareLibService implements Service, Instrumentable {
return shareLibMap;
}
+ private Map<String, Map<Path, Path>> getSymlinkMapping() {
+ return symlinkMapping;
+ }
+
/**
- * Gets the action system lib common jars.
+ * Gets the action sharelib lib jars.
*
- * @param actionKey the action key
+ * @param shareLibKey the sharelib key
* @return List of paths
* @throws IOException
*/
- public List<Path> getShareLibJars(String actionKey) throws IOException {
+ public List<Path> getShareLibJars(String shareLibKey) throws IOException {
// Sharelib map is empty means that on previous or startup attempt of
// caching sharelib has failed.Trying to reload
if (shareLibMap.isEmpty() && !shareLibLoadAttempted) {
@@ -308,18 +329,45 @@ public class ShareLibService implements Service, Instrumentable {
}
}
}
- return shareLibMap.get(actionKey);
+ checkSymlink(shareLibKey);
+ return shareLibMap.get(shareLibKey);
+ }
+
+ private void checkSymlink(String shareLibKey) throws IOException {
+ if (!HadoopShims.isSymlinkSupported() || symlinkMapping.get(shareLibKey) == null
+ || symlinkMapping.get(shareLibKey).isEmpty()) {
+ return;
+ }
+
+ HadoopShims fileSystem = new HadoopShims(fs);
+ for (Path path : symlinkMapping.get(shareLibKey).keySet()) {
+ if (!symlinkMapping.get(shareLibKey).get(path).equals(fileSystem.getSymLinkTarget(path))) {
+ synchronized (ShareLibService.class) {
+ Map<String, List<Path>> tempShareLibMap = new HashMap<String, List<Path>>(shareLibMap);
+ Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>(symlinkMapping);
+
+ LOG.info(MessageFormat.format("Symlink target for [{0}] has changed, was [{1}], now [{2}]", shareLibKey,
+ path, fileSystem.getSymLinkTarget(path)));
+ loadShareLibMetaFile(tempShareLibMap, tmpSymlinkMapping, sharelibMappingFile, shareLibKey);
+ shareLibMap = tempShareLibMap;
+ symlinkMapping = tmpSymlinkMapping;
+ return;
+ }
+
+ }
+ }
+
}
/**
* Gets the launcher jars.
*
- * @param actionKey the action key
+ * @param shareLibKey the shareLib key
* @return launcher jars paths
* @throws ClassNotFoundException
* @throws IOException
*/
- public List<Path> getSystemLibJars(String actionKey) throws IOException {
+ public List<Path> getSystemLibJars(String shareLibKey) throws IOException {
List<Path> returnList = new ArrayList<Path>();
// Sharelib map is empty means that on previous or startup attempt of
// caching launcher jars has failed.Trying to reload
@@ -331,10 +379,10 @@ public class ShareLibService implements Service, Instrumentable {
}
}
}
- returnList.addAll(launcherLibMap.get(actionKey));
+ returnList.addAll(launcherLibMap.get(shareLibKey));
}
- if (actionKey.equals(JavaActionExecutor.OOZIE_COMMON_LIBDIR)) {
- List<Path> sharelibList = getShareLibJars(actionKey);
+ if (shareLibKey.equals(JavaActionExecutor.OOZIE_COMMON_LIBDIR)) {
+ List<Path> sharelibList = getShareLibJars(shareLibKey);
if (sharelibList != null) {
returnList.addAll(sharelibList);
}
@@ -422,7 +470,7 @@ public class ShareLibService implements Service, Instrumentable {
}
});
- //Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days.
+ // Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days.
// refer OOZIE-1761
for (int i = 1; i < dirList.length; i++) {
Path dirPath = dirList[i].getPath();
@@ -460,11 +508,12 @@ public class ShareLibService implements Service, Instrumentable {
}
Map<String, List<Path>> tempShareLibMap = new HashMap<String, List<Path>>();
+ Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>();
if (!StringUtils.isEmpty(sharelibMappingFile.trim())) {
- String sharelibMetaFileNewTimeStamp = JsonUtils.formatDateRfc822(new Date(fs.getFileStatus(
- new Path(sharelibMappingFile)).getModificationTime()),"GMT");
- loadShareLibMetaFile(tempShareLibMap, sharelibMappingFile);
+ String sharelibMetaFileNewTimeStamp = JsonUtils.formatDateRfc822(
+ new Date(fs.getFileStatus(new Path(sharelibMappingFile)).getModificationTime()), "GMT");
+ loadShareLibMetaFile(tempShareLibMap, tmpSymlinkMapping, sharelibMappingFile, null);
status.put("sharelibMetaFile", sharelibMappingFile);
status.put("sharelibMetaFileNewTimeStamp", sharelibMetaFileNewTimeStamp);
status.put("sharelibMetaFileOldTimeStamp", sharelibMetaFileOldTimeStamp);
@@ -483,12 +532,12 @@ public class ShareLibService implements Service, Instrumentable {
}
shareLibMap = tempShareLibMap;
+ symlinkMapping = tmpSymlinkMapping;
return status;
}
/**
- * Update share lib cache. Parse the share lib directory and each sub
- * directory is a action key
+ * Update share lib cache. Parse the share lib directory and each sub directory is a action key
*
* @param shareLibMap the share lib jar map
* @param shareLibpath the share libpath
@@ -522,16 +571,17 @@ public class ShareLibService implements Service, Instrumentable {
}
/**
- * Load share lib text file. Sharelib mapping files contains list of
- * key=value. where key is the action key and value is the DFS location of
- * sharelib files.
+ * Load share lib text file. Sharelib mapping files contains list of key=value. where key is the action key and
+ * value is the DFS location of sharelib files.
*
* @param shareLibMap the share lib jar map
* @param sharelipFileMapping the sharelip file mapping
+ * @param symlinkMapping the symlink mapping
+ * @parm shareLibKey the sharelib key
* @throws IOException Signals that an I/O exception has occurred.
*/
- @SuppressWarnings("unchecked")
- private void loadShareLibMetaFile(Map<String, List<Path>> shareLibMap, String sharelibFileMapping)
+ private void loadShareLibMetaFile(Map<String, List<Path>> shareLibMap,
+ Map<String, Map<Path, Path>> symlinkMapping, String sharelibFileMapping, String shareLibKey)
throws IOException {
Path shareFileMappingPath = new Path(sharelibFileMapping);
@@ -542,22 +592,34 @@ public class ShareLibService implements Service, Instrumentable {
for (Object keyObject : prop.keySet()) {
String key = (String) keyObject;
- if (key.toLowerCase().startsWith(SHARE_LIB_CONF_PREFIX)) {
- String mapKey = key.substring(SHARE_LIB_CONF_PREFIX.length() + 1);
- String pathList[] = ((String) prop.get(key)).split(",");
- List<Path> listOfPaths = new ArrayList<Path>();
- for (String dfsPath : pathList) {
- getPathRecursively(fs, new Path(dfsPath), listOfPaths);
- }
- shareLibMap.put(mapKey, listOfPaths);
- LOG.info("Share lib for " + mapKey + ":" + listOfPaths);
-
- }
- else {
- LOG.info(" Not adding " + key + " to sharelib, not prefix with " + SHARE_LIB_CONF_PREFIX);
+ String mapKey = key.substring(SHARE_LIB_CONF_PREFIX.length() + 1);
+ if (key.toLowerCase().startsWith(SHARE_LIB_CONF_PREFIX)
+ && (shareLibKey == null || shareLibKey.equals(mapKey))) {
+ loadSharelib(shareLibMap, symlinkMapping, mapKey, ((String) prop.get(key)).split(","));
}
+ }
+ }
+
+ private void loadSharelib(Map<String, List<Path>> tmpShareLibMap, Map<String, Map<Path, Path>> tmpSymlinkMapping,
+ String shareLibKey, String pathList[]) throws IOException {
+ List<Path> listOfPaths = new ArrayList<Path>();
+ Map<Path, Path> symlinkMappingforAction = new HashMap<Path, Path>();
+ HadoopShims fileSystem = new HadoopShims(fs);
+
+ for (String dfsPath : pathList) {
+ Path path = new Path(dfsPath);
+ getPathRecursively(fs, path, listOfPaths);
+ if (HadoopShims.isSymlinkSupported() && fileSystem.isSymlink(path)) {
+ symlinkMappingforAction.put(path, fileSystem.getSymLinkTarget(path));
+ }
+ }
+ if (HadoopShims.isSymlinkSupported()) {
+ LOG.info("symlink for " + shareLibKey + ":" + symlinkMappingforAction);
+ tmpSymlinkMapping.put(shareLibKey, symlinkMappingforAction);
}
+ tmpShareLibMap.put(shareLibKey, listOfPaths);
+ LOG.info("Share lib for " + shareLibKey + ":" + listOfPaths);
}
/**
@@ -683,6 +745,27 @@ public class ShareLibService implements Service, Instrumentable {
return getLauncherlibPath().toUri().toString();
}
});
+ instr.addVariable("libs", "sharelib.symlink.mapping", new Instrumentation.Variable<String>() {
+ @Override
+ public String getValue() {
+ Map<String, Map<Path, Path>> shareLibSymlinkMapping = getSymlinkMapping();
+ if (shareLibSymlinkMapping != null && !shareLibSymlinkMapping.isEmpty()
+ && shareLibSymlinkMapping.values() != null && !shareLibSymlinkMapping.values().isEmpty()) {
+ StringBuffer bf = new StringBuffer();
+ for (Entry<String, Map<Path, Path>> key : shareLibSymlinkMapping.entrySet())
+ if (key.getKey() != null && !key.getValue().isEmpty()) {
+ for (Path path : key.getValue().keySet()) {
+ bf.append(path).append("(").append(key).append(")").append("=>")
+ .append(shareLibSymlinkMapping.get(key).get(path)).append(",");
+ }
+
+ }
+ return bf.toString();
+ }
+ return "(none)";
+ }
+ });
+
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index 67bb2e5..f261448 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -40,6 +40,7 @@ import org.apache.oozie.action.hadoop.JavaActionExecutor;
import org.apache.oozie.action.hadoop.PigActionExecutor;
import org.apache.oozie.action.hadoop.TestJavaActionExecutor;
import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.hadoop.utils.HadoopShims;
import org.apache.oozie.test.XFsTestCase;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.XConfiguration;
@@ -107,15 +108,16 @@ public class TestShareLibService extends XFsTestCase {
setSystemProps();
Configuration conf = services.getConf();
conf.set(ShareLibService.FAIL_FAST_ON_STARTUP, "true");
- //Set dummyfile as metafile which doesn't exist.
+ // Set dummyfile as metafile which doesn't exist.
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, String.valueOf(new Date().getTime()));
try {
services.init();
fail("Should throw exception");
}
- catch(Throwable e){
+ catch (Throwable e) {
assertTrue(e.getMessage().contains("E0104: Could not fully initialize service"));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -134,7 +136,8 @@ public class TestShareLibService extends XFsTestCase {
assertTrue(getFileSystem().exists(launcherPath.get(0)));
List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig");
assertTrue(getFileSystem().exists(pigLauncherPath.get(0)));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -173,7 +176,8 @@ public class TestShareLibService extends XFsTestCase {
else {
assertEquals(2, cacheFiles.length);
}
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -211,7 +215,8 @@ public class TestShareLibService extends XFsTestCase {
else {
assertEquals(2, cacheFiles.length);
}
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -271,7 +276,8 @@ public class TestShareLibService extends XFsTestCase {
else {
assertEquals(3, cacheFiles.length);
}
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -344,7 +350,8 @@ public class TestShareLibService extends XFsTestCase {
assertTrue(fs.exists(noexpirePath));
assertTrue(fs.exists(noexpirePath1));
assertTrue(fs.exists(expirePath));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -389,7 +396,8 @@ public class TestShareLibService extends XFsTestCase {
assertTrue(fs.exists(noexpirePath1));
assertTrue(fs.exists(expirePath));
assertFalse(fs.exists(expirePath1));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -401,7 +409,7 @@ public class TestShareLibService extends XFsTestCase {
FileSystem fs = getFileSystem();
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
- //Use basepath if there is no timestamped directory
+ // Use basepath if there is no timestamped directory
fs.mkdirs(basePath);
Path pigPath = new Path(basePath.toString() + Path.SEPARATOR + "pig");
fs.mkdirs(pigPath);
@@ -409,7 +417,8 @@ public class TestShareLibService extends XFsTestCase {
services.init();
ShareLibService shareLibService = Services.get().get(ShareLibService.class);
assertNotNull(shareLibService.getShareLibJars("pig"));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -421,7 +430,7 @@ public class TestShareLibService extends XFsTestCase {
FileSystem fs = getFileSystem();
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
- //Use timedstamped directory if available
+ // Use timedstamped directory if available
Date time = new Date(System.currentTimeMillis());
Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time));
fs.mkdirs(libpath);
@@ -439,7 +448,8 @@ public class TestShareLibService extends XFsTestCase {
assertNotNull(shareLibService.getShareLibJars("pig_9"));
assertNotNull(shareLibService.getShareLibJars("pig_10"));
assertNull(shareLibService.getShareLibJars("pig_11"));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -464,7 +474,8 @@ public class TestShareLibService extends XFsTestCase {
services.init();
ShareLibService shareLibService = Services.get().get(ShareLibService.class);
assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -474,7 +485,7 @@ public class TestShareLibService extends XFsTestCase {
services = new Services();
FileSystem fs = getFileSystem();
setSystemProps();
- creatTempshareLibKeymap(fs);
+ createTestShareLibMetaFile(fs);
Configuration conf = services.getConf();
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
@@ -484,7 +495,8 @@ public class TestShareLibService extends XFsTestCase {
assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
fs.delete(new Path("shareLibPath/"), true);
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -499,7 +511,8 @@ public class TestShareLibService extends XFsTestCase {
Date time = new Date(System.currentTimeMillis());
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
- Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time));
+ Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX
+ + ShareLibService.dateFormat.format(time));
fs.mkdirs(libpath);
Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
@@ -542,7 +555,8 @@ public class TestShareLibService extends XFsTestCase {
else {
assertEquals(2, cacheFiles.length);
}
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -551,7 +565,7 @@ public class TestShareLibService extends XFsTestCase {
public void testShareLibLoadFileMultipleFile() throws Exception {
FileSystem fs = getFileSystem();
services = new Services();
- creatTempshareLibKeymap_multipleFile(fs);
+ createTestShareLibMetaFile_multipleFile(fs);
setSystemProps();
Configuration conf = services.getConf();
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
@@ -562,7 +576,8 @@ public class TestShareLibService extends XFsTestCase {
assertNull(shareLibService.getShareLibJars("something_new"));
assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
fs.delete(new Path("shareLibPath/"), true);
- } finally {
+ }
+ finally {
services.destroy();
}
}
@@ -574,10 +589,8 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
try {
- services.init();
FileSystem fs = getFileSystem();
Date time = new Date(System.currentTimeMillis());
- ShareLibService shareLibService = Services.get().get(ShareLibService.class);
Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX
+ ShareLibService.dateFormat.format(time));
@@ -585,6 +598,8 @@ public class TestShareLibService extends XFsTestCase {
Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
fs.mkdirs(ooziePath);
createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
+ services.init();
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
shareLibService.init(services);
List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
assertEquals(launcherPath.size(), 2);
@@ -596,13 +611,70 @@ public class TestShareLibService extends XFsTestCase {
}
}
- public void createFile(String filename) throws IOException {
+ @Test
+ public void testMetafileSymlink() throws ServiceException, IOException {
+ // Assume.assumeTrue("Skipping for hadoop - 1.x",HadoopFileSystem.isSymlinkSupported());
+ if (!HadoopShims.isSymlinkSupported()) {
+ return;
+ }
+
+ services = new Services();
+ setSystemProps();
+ Configuration conf = services.getConf();
+ conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+ services.init();
+ FileSystem fs = getFileSystem();
+ Properties prop = new Properties();
+ try {
+
+ String testPath = "shareLibPath/";
+
+ Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
+ Path basePath1 = new Path(testPath + Path.SEPARATOR + "testPath1");
+ Path symlink = new Path("symlink/");
+ fs.mkdirs(basePath);
+
+ createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
+ createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_1.jar");
+
+ createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_2.jar");
+ createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_3.jar");
+ createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_4.jar");
+
+ HadoopShims fileSystem = new HadoopShims(fs);
+ fileSystem.createSymlink(basePath, symlink, true);
+
+ prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + symlink.toString());
+ createTestShareLibMetaFile(fs, prop);
+ assertEquals(fileSystem.isSymlink(symlink), true);
+
+ conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
+ conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+ try {
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+ shareLibService.init(services);
+ assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
+ new HadoopShims(fs).createSymlink(basePath1, symlink, true);
+ assertEquals(shareLibService.getShareLibJars("pig").size(), 3);
+ }
+ finally {
+ fs.delete(new Path("shareLibPath/"), true);
+ fs.delete(symlink, true);
+ services.destroy();
+ }
+ }
+ catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ private void createFile(String filename) throws IOException {
Path path = new Path(filename);
FSDataOutputStream out = getFileSystem().create(path);
out.close();
}
- public void creatTempshareLibKeymap(FileSystem fs) {
+ private void createTestShareLibMetaFile(FileSystem fs) {
Properties prop = new Properties();
try {
@@ -620,9 +692,17 @@ public class TestShareLibService extends XFsTestCase {
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString());
prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new", "/user/test/" + somethingNew.toString());
+ createTestShareLibMetaFile(fs, prop);
- FSDataOutputStream out = fs.create(new Path("/user/test/config.properties"));
+ }
+ catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+ private void createTestShareLibMetaFile(FileSystem fs, Properties prop) {
+ try {
+ FSDataOutputStream out = fs.create(new Path("/user/test/config.properties"));
prop.store(out, null);
out.close();
@@ -632,7 +712,7 @@ public class TestShareLibService extends XFsTestCase {
}
}
- public void creatTempshareLibKeymap_multipleFile(FileSystem fs) {
+ public void createTestShareLibMetaFile_multipleFile(FileSystem fs) {
Properties prop = new Properties();
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
new file mode 100644
index 0000000..9a19770
--- /dev/null
+++ b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.hadoop.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import java.io.IOException;
+
+public class HadoopShims {
+ FileSystem fs;
+
+ public HadoopShims(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ public static boolean isSymlinkSupported() {
+ return false;
+ }
+
+ public Path getSymLinkTarget(Path p) throws IOException {
+ return p;
+ }
+
+ public boolean isSymlink(Path p) throws IOException {
+ return false;
+ }
+
+ public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
new file mode 100644
index 0000000..bea4781
--- /dev/null
+++ b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.hadoop.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import java.io.IOException;
+
+
+public class HadoopShims {
+ FileSystem fs;
+
+ public HadoopShims(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ public static boolean isSymlinkSupported() {
+ return false;
+ }
+
+ public Path getSymLinkTarget(Path p) throws IOException {
+ return p;
+ }
+
+ public boolean isSymlink(Path p) throws IOException {
+ return false;
+ }
+
+ public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
new file mode 100644
index 0000000..acebd60
--- /dev/null
+++ b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.hadoop.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import java.io.IOException;
+
+public class HadoopShims {
+ FileSystem fs;
+
+ public HadoopShims(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ public static boolean isSymlinkSupported() {
+ return true;
+ }
+
+ public Path getSymLinkTarget(Path p) throws IOException {
+ return fs.getFileLinkStatus(p).getSymlink();
+ }
+
+ public boolean isSymlink(Path p) throws IOException {
+ return fs.getFileLinkStatus(p).isSymlink();
+ }
+
+ public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
+ fs.createSymlink(target, link, createParent);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
----------------------------------------------------------------------
diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
new file mode 100644
index 0000000..acebd60
--- /dev/null
+++ b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.hadoop.utils;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import java.io.IOException;
+
+public class HadoopShims {
+ FileSystem fs;
+
+ public HadoopShims(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ public static boolean isSymlinkSupported() {
+ return true;
+ }
+
+ public Path getSymLinkTarget(Path p) throws IOException {
+ return fs.getFileLinkStatus(p).getSymlink();
+ }
+
+ public boolean isSymlink(Path p) throws IOException {
+ return fs.getFileLinkStatus(p).isSymlink();
+ }
+
+ public void createSymlink(Path target, Path link, boolean createParent) throws IOException {
+ fs.createSymlink(target, link, createParent);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/b5d409e6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index aa8ef33..e3dad74 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-2045 Symlink support for sharelib (puru)
OOZIE-1385 Make Uber Mode the default (rkanter)
OOZIE-1890 Make oozie-site empty and reconcile defaults between oozie-default and the code (seoeun25 via rkanter)
OOZIE-2001 Workflow re-runs doesn't update coord action status (jaydeepvishwakarma via shwethags)