You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by as...@apache.org on 2017/07/20 16:58:06 UTC

[1/2] oozie git commit: OOZIE-2601 Ability to use local paths for the sharelib

Repository: oozie
Updated Branches:
  refs/heads/master 18694b616 -> 25a8b99d5


http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index 390545d..a4f3919 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -26,14 +26,18 @@ import java.net.URLDecoder;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.io.Files;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.oozie.WorkflowActionBean;
 import org.apache.oozie.WorkflowJobBean;
@@ -54,23 +58,28 @@ import org.junit.Test;
 import com.google.common.collect.Lists;
 
 public class TestShareLibService extends XFsTestCase {
+    private static final String HDFS_SCHEME_PREFIX = "hdfs";
+    private static final String TEST_HDFS_HOME = "/user/test/";
+    private static final String TEST_MAPPING_FILENAME = "config.properties";
+    private static final String TEST_HDFS_MAPPING_FILE_PATH = TEST_HDFS_HOME + TEST_MAPPING_FILENAME;
+    private static final String SHARELIB_PATH = "shareLibPath/";
 
     Services services;
     private static String testCaseDirPath;
-    String shareLibPath = "shareLibPath";
     SimpleDateFormat dt = new SimpleDateFormat("yyyyMMddHHmmss");
-    final String sharelibPath = "sharelib";
-    final String metaFile = "/user/test/config.properties";
 
     @Override
     protected void setUp() throws Exception {
         super.setUp();
         testCaseDirPath = getTestCaseDir();
+        services = new Services();
+        setSystemProps();
     }
 
     @Override
     protected void tearDown() throws Exception {
         super.tearDown();
+        services.destroy();
     }
 
     private void setSystemProps() throws IOException {
@@ -78,22 +87,21 @@ public class TestShareLibService extends XFsTestCase {
         IOUtils.createJar(new File(getTestCaseDir()), MyPig.class.getName() + ".jar", MyPig.class);
         IOUtils.createJar(new File(getTestCaseDir()), TestHive.class.getName() + ".jar", TestHive.class);
 
-        Configuration conf = services.get(ConfigurationService.class).getConf();
+        Configuration conf = getOozieConfig();
         conf.set(WorkflowAppService.SYSTEM_LIB_PATH, getFsTestCaseDir() + "/share/lib");
         conf.set(Services.CONF_SERVICE_CLASSES, conf.get(Services.CONF_SERVICE_CLASSES) + ","
                 + DummyShareLibService.class.getName());
         conf.setStrings(ActionService.CONF_ACTION_EXECUTOR_CLASSES, DummyPigActionExecutor.class.getName(),
                 DummyHiveActionExecutor.class.getName());
-
     }
 
     public static class DummyShareLibService extends ShareLibService {
         @Override
         public String findContainingJar(Class<?> clazz) {
             if (JavaActionExecutor.getCommonLauncherClasses().contains(clazz)) {
-                return testCaseDirPath + "/" + MyOozie.class.getName() + ".jar";
+                return testCaseDirPath + Path.SEPARATOR + MyOozie.class.getName() + ".jar";
             }
-            return testCaseDirPath + "/" + clazz.getName() + ".jar";
+            return testCaseDirPath + Path.SEPARATOR + clazz.getName() + ".jar";
         }
     }
 
@@ -128,9 +136,7 @@ public class TestShareLibService extends XFsTestCase {
 
     @Test
     public void testfailFast() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
+        Configuration conf = getOozieConfig();
         conf.set(ShareLibService.FAIL_FAST_ON_STARTUP, "true");
         // Set dummyfile as metafile which doesn't exist.
         conf.set(ShareLibService.SHARELIB_MAPPING_FILE, String.valueOf(new Date().getTime()));
@@ -141,99 +147,68 @@ public class TestShareLibService extends XFsTestCase {
         catch (Throwable e) {
             assertTrue(e.getMessage().contains("E0104: Could not fully initialize service"));
         }
-        finally {
-            services.destroy();
-        }
     }
 
     @Test
     public void testCreateLauncherLibPath() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
-        try {
-            services.init();
-            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
-            assertNotNull(launcherPath);
-            assertTrue(getFileSystem().exists(launcherPath.get(0)));
-            List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig");
-            assertTrue(getFileSystem().exists(pigLauncherPath.get(0)));
-        }
-        finally {
-            services.destroy();
-        }
+        setShipLauncherInOozieConfig();
+        services.init();
+        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+        List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
+        assertNotNull(launcherPath);
+        assertTrue(getFileSystem().exists(launcherPath.get(0)));
+        List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig");
+        assertTrue(getFileSystem().exists(pigLauncherPath.get(0)));
     }
 
     @Test
     public void testAddShareLibDistributedCache() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
 
-        try {
-            services.init();
-            String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
-                    + getNameNodeUri() + "</name-node>" + "</java>";
-            Element eActionXml = XmlUtils.parseXml(actionXml);
-            XConfiguration protoConf = new XConfiguration();
-            protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-            WorkflowJobBean wfj = new WorkflowJobBean();
-            wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
-            wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
-            Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
-            PigActionExecutor ae = new PigActionExecutor();
-            Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
-            ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
-            verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), MyPig.class.getName() + ".jar",
-                    MyOozie.class.getName() + ".jar");
-        }
-        finally {
-            services.destroy();
-        }
+        services.init();
+        String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+                + getNameNodeUri() + "</name-node>" + "</java>";
+        Element eActionXml = XmlUtils.parseXml(actionXml);
+        XConfiguration protoConf = new XConfiguration();
+        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+        WorkflowJobBean wfj = new WorkflowJobBean();
+        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+        wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
+        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+        PigActionExecutor ae = new PigActionExecutor();
+        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+        verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), MyPig.class.getName() + ".jar",
+                MyOozie.class.getName() + ".jar");
     }
 
     @Test
     public void testAddShareLib_pig() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
-        try {
-            services.init();
-            String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
-                    + getNameNodeUri() + "</name-node>" + "</pig>";
-            Element eActionXml = XmlUtils.parseXml(actionXml);
-            XConfiguration protoConf = new XConfiguration();
-            protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-            WorkflowJobBean wfj = new WorkflowJobBean();
-            wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
-            wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
-            Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
-            PigActionExecutor ae = new PigActionExecutor();
-            Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
-            ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
-            verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "MyPig.jar", "MyOozie.jar");
-
-        }
-        finally {
-            services.destroy();
-        }
+        setShipLauncherInOozieConfig();
+        services.init();
+        String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+                + getNameNodeUri() + "</name-node>" + "</pig>";
+        Element eActionXml = XmlUtils.parseXml(actionXml);
+        XConfiguration protoConf = new XConfiguration();
+        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+        WorkflowJobBean wfj = new WorkflowJobBean();
+        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+        wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
+        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+        PigActionExecutor ae = new PigActionExecutor();
+        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+        verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "MyPig.jar", "MyOozie.jar");
     }
 
     @Test
     public void testAddShareLib_pig_withVersion() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
 
         FileSystem fs = getFileSystem();
         Date time = new Date(System.currentTimeMillis());
 
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
         Path libpath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));
         fs.mkdirs(libpath);
@@ -245,46 +220,37 @@ public class TestShareLibService extends XFsTestCase {
         fs.mkdirs(pigPath1);
         fs.mkdirs(pigPath2);
 
-        createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
+        createFiles(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
 
-        try {
-            services.init();
-            String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
-                    + getNameNodeUri() + "</name-node>"
-                    + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
-            Element eActionXml = XmlUtils.parseXml(actionXml);
-            XConfiguration protoConf = new XConfiguration();
-            protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-            WorkflowJobBean wfj = new WorkflowJobBean();
-            protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
-            wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
-            wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
-
-            Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
-            PigActionExecutor ae = new PigActionExecutor();
-            Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
-            jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
-            ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+        services.init();
+        String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+                + getNameNodeUri() + "</name-node>"
+                + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
+        Element eActionXml = XmlUtils.parseXml(actionXml);
+        XConfiguration protoConf = new XConfiguration();
+        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+        WorkflowJobBean wfj = new WorkflowJobBean();
+        protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+        wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
 
-            verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "MyPig.jar", "MyOozie.jar",
-                    "pig-10.jar");
+        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+        PigActionExecutor ae = new PigActionExecutor();
+        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+        jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
+        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
 
-        }
-        finally {
-            services.destroy();
-        }
+        verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "MyPig.jar", "MyOozie.jar",
+                "pig-10.jar");
     }
 
     @Test
     public void testPurgeShareLib() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
         FileSystem fs = getFileSystem();
         long expiryTime = System.currentTimeMillis()
                 - TimeUnit.MILLISECONDS.convert(
-                        services.get(ConfigurationService.class).getConf()
+                        getOozieConfig()
                                 .getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS);
 
         // for directory created 8 days back to be deleted
@@ -294,36 +260,29 @@ public class TestShareLibService extends XFsTestCase {
         // for directory created 5 days back NOT to be deleted
         String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
 
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
         Path expirePath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + expireTs);
         Path noexpirePath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + noexpireTs);
         Path noexpirePath1 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + noexpireTs1);
 
         createDirs(fs, expirePath, noexpirePath, noexpirePath1);
-        try {
-            services.init();
-            assertEquals(4, fs.listStatus(basePath).length);
-            assertTrue(fs.exists(noexpirePath));
-            assertTrue(fs.exists(noexpirePath1));
-            assertTrue(fs.exists(expirePath));
-        }
-        finally {
-            services.destroy();
-        }
+
+        services.init();
+        assertEquals(4, fs.listStatus(basePath).length);
+        assertTrue(fs.exists(noexpirePath));
+        assertTrue(fs.exists(noexpirePath1));
+        assertTrue(fs.exists(expirePath));
     }
 
     @Test
     public void testPurgeLauncherJar() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
         FileSystem fs = getFileSystem();
 
         long expiryTime = System.currentTimeMillis()
                 - TimeUnit.MILLISECONDS.convert(
-                        services.get(ConfigurationService.class).getConf()
+                        getOozieConfig()
                                 .getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS);
 
         // for directory created 8 days back to be deleted
@@ -333,7 +292,7 @@ public class TestShareLibService extends XFsTestCase {
         // for directory created 5 days back NOT to be deleted
         String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
 
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
         Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_LIB_PREFIX + expireTs);
         Path noexpirePath = new Path(basePath, ShareLibService.LAUNCHER_LIB_PREFIX + noexpireTs);
@@ -341,38 +300,30 @@ public class TestShareLibService extends XFsTestCase {
         Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_LIB_PREFIX + noexpireTs1);
         createDirs(fs, expirePath, noexpirePath, noexpirePath1);
 
-        try {
-            services.init();
-            assertEquals(4, fs.listStatus(basePath).length);
-            assertTrue(fs.exists(noexpirePath));
-            assertTrue(fs.exists(noexpirePath1));
-            assertTrue(fs.exists(expirePath));
-        }
-        finally {
-            services.destroy();
-        }
+        services.init();
+        assertEquals(4, fs.listStatus(basePath).length);
+        assertTrue(fs.exists(noexpirePath));
+        assertTrue(fs.exists(noexpirePath1));
+        assertTrue(fs.exists(expirePath));
     }
 
     // Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days.
     // refer OOZIE-1761
     @Test
     public void testPurgeJar() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
         final FileSystem fs = getFileSystem();
         // for directory created 8 days back to be deleted
         long expiryTime = System.currentTimeMillis()
                 - TimeUnit.MILLISECONDS.convert(
-                        services.get(ConfigurationService.class).getConf()
+                        getOozieConfig()
                                 .getInt(ShareLibService.LAUNCHERJAR_LIB_RETENTION, 7), TimeUnit.DAYS);
 
         String expireTs = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
         String expireTs1 = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
         String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
         String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
-        final Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        final Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
 
         Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_LIB_PREFIX + expireTs);
@@ -381,54 +332,42 @@ public class TestShareLibService extends XFsTestCase {
         Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_LIB_PREFIX + noexpireTs1);
 
         createDirs(fs, expirePath, expirePath1, noexpirePath, noexpirePath1);
-        try {
-            services.init();
-            // Wait for the scheduled purge runnable to complete
-            waitFor(20 * 1000, new Predicate() {
-                @Override
-                public boolean evaluate() throws Exception {
-                    return (fs.listStatus(basePath).length == 4);
-                }
-            });
-            assertEquals(4, fs.listStatus(basePath).length);
-            assertTrue(fs.exists(noexpirePath));
-            assertTrue(fs.exists(noexpirePath1));
-            assertTrue(fs.exists(expirePath));
-            assertFalse(fs.exists(expirePath1));
-        }
-        finally {
-            services.destroy();
-        }
+
+        services.init();
+        // Wait for the scheduled purge runnable to complete
+        waitFor(20 * 1000, new Predicate() {
+            @Override
+            public boolean evaluate() throws Exception {
+                return (fs.listStatus(basePath).length == 4);
+            }
+        });
+        assertEquals(4, fs.listStatus(basePath).length);
+        assertTrue(fs.exists(noexpirePath));
+        assertTrue(fs.exists(noexpirePath1));
+        assertTrue(fs.exists(expirePath));
+        assertFalse(fs.exists(expirePath1));
     }
 
     @Test
     public void testGetShareLibCompatible() throws Exception {
-        services = new Services();
-        setSystemProps();
         FileSystem fs = getFileSystem();
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
 
         // Use basepath if there is no timestamped directory
         fs.mkdirs(basePath);
         Path pigPath = new Path(basePath.toString() + Path.SEPARATOR + "pig");
         fs.mkdirs(pigPath);
-        try {
-            services.init();
-            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            assertNotNull(shareLibService.getShareLibJars("pig"));
-        }
-        finally {
-            services.destroy();
-        }
+
+        services.init();
+        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+        assertNotNull(shareLibService.getShareLibJars("pig"));
     }
 
     @Test
     public void testGetShareLibPath() throws Exception {
-        services = new Services();
-        setSystemProps();
         FileSystem fs = getFileSystem();
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
 
         // Use timedstamped directory if available
@@ -442,191 +381,184 @@ public class TestShareLibService extends XFsTestCase {
         fs.mkdirs(pigPath);
         fs.mkdirs(pigPath1);
         fs.mkdirs(pigPath2);
-        try {
-            services.init();
-            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            assertNotNull(shareLibService.getShareLibJars("pig"));
-            assertNotNull(shareLibService.getShareLibJars("pig_9"));
-            assertNotNull(shareLibService.getShareLibJars("pig_10"));
-            assertNull(shareLibService.getShareLibJars("pig_11"));
-        }
-        finally {
-            services.destroy();
-        }
+
+        services.init();
+        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+        assertNotNull(shareLibService.getShareLibJars("pig"));
+        assertNotNull(shareLibService.getShareLibJars("pig_9"));
+        assertNotNull(shareLibService.getShareLibJars("pig_10"));
+        assertNull(shareLibService.getShareLibJars("pig_11"));
     }
 
     @Test
     public void testShareLib() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
         FileSystem fs = getFileSystem();
         String dir1 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
         String dir2 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
         String dir3 = dt.format(new Date(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(3, TimeUnit.DAYS)));
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
         Path path1 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + dir1);
         Path path2 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + dir2);
         Path path3 = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + dir3);
         createDirs(fs, path1, path2, path3);
-        createFile(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
+        createFiles(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
+
+        services.init();
+        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+        assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
+    }
+
+    private void validateShareLibLoadFiles(final FileSystem fs, final String schema) throws Exception {
         try {
             services.init();
             ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
+
+            verifyShareLibFromMappingFileContent(schema, shareLibService);
         }
         finally {
-            services.destroy();
+            if (schema.startsWith(HDFS_SCHEME_PREFIX)) {
+                fs.delete(new Path(SHARELIB_PATH), true);
+                fs.delete(new Path("linkFile.xml"), true);
+            }
+        }
+    }
+
+    private void verifyShareLibFromMappingFileContent(String schema, ShareLibService shareLibService) throws IOException
+    {
+        assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
+        assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
+        assertTrue(shareLibService.getShareLibJars("directjar").get(0).getName().endsWith("direct.jar"));
+        assertTrue(shareLibService.getShareLibJars("linkFile").get(0).getName().endsWith("targetOfLinkFile.xml"));
+
+        List<Path> listOfPaths = shareLibService.getShareLibJars("directjar");
+        for (Path p : listOfPaths) {
+            assertTrue(p.toString().startsWith(schema));
         }
     }
 
+    private void setupShareLibLoadFiles(FileSystem fs, String testUserHome) throws ServiceException, IOException {
+        createShareLibMetaFileTestResources(fs, testUserHome);
+        setShipLauncherInOozieConfig();
+        Configuration conf = getOozieConfig();
+        conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + testUserHome + TEST_MAPPING_FILENAME);
+    }
+
     @Test
-    public void testShareLibLoadFile() throws Exception {
-        services = new Services();
-        FileSystem fs = getFileSystem();
-        setSystemProps();
-        createTestShareLibMetaFile(fs);
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+    public void testShareLibLoadFilesFromLocalFs() throws Exception {
+        final FileSystem localFs = newLocalFileSystem();
+        final String testUserHome = Files.createTempDir().toString() + Path.SEPARATOR;
         try {
-            services.init();
-            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
-            assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
-            assertTrue(shareLibService.getShareLibJars("directjar").get(0).getName().endsWith("direct.jar"));
-            assertTrue(shareLibService.getShareLibJars("linkFile").get(0).getName().endsWith("targetOfLinkFile.xml"));
-
-            List<Path> listOfPaths = shareLibService.getShareLibJars("directjar");
-            for (Path p : listOfPaths) {
-                assertTrue(p.toString().startsWith("hdfs"));
-            }
-            fs.delete(new Path("shareLibPath/"), true);
+            setupShareLibLoadFiles(localFs, testUserHome);
+            validateShareLibLoadFiles(localFs, FSUtils.FILE_SCHEME_PREFIX);
         }
         finally {
-            services.destroy();
+            localFs.delete(new Path(testUserHome), true);
         }
     }
 
     @Test
+    public void testShareLibLoadFilesFromHDFS() throws Exception {
+        FileSystem fs = getFileSystem();
+        setupShareLibLoadFiles(fs, TEST_HDFS_HOME);
+        validateShareLibLoadFiles(fs, HDFS_SCHEME_PREFIX);
+    }
+
+    @Test
     public void testLoadfromDFS() throws Exception {
-        services = new Services();
-        setSystemProps();
-        try {
-            services.init();
-            FileSystem fs = getFileSystem();
-            Date time = new Date(System.currentTimeMillis());
-
-            Path basePath = new Path(services.get(ConfigurationService.class).getConf()
-                    .get(WorkflowAppService.SYSTEM_LIB_PATH));
-            Path libpath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX
-                    + ShareLibService.dateFormat.format(time));
-            fs.mkdirs(libpath);
-
-            Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
-            Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
-            Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
-            Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
-            fs.mkdirs(pigPath);
-            fs.mkdirs(ooziePath);
-            fs.mkdirs(pigPath1);
-            fs.mkdirs(pigPath2);
-
-            createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
-            createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
+        services.init();
+        FileSystem fs = getFileSystem();
+        Date time = new Date(System.currentTimeMillis());
 
-            String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
-                    + getNameNodeUri() + "</name-node>"
-                    + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
-            Element eActionXml = XmlUtils.parseXml(actionXml);
-            XConfiguration protoConf = new XConfiguration();
-            protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
-            WorkflowJobBean wfj = new WorkflowJobBean();
-            protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
-            wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
-            wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
+        Path basePath = new Path(getOozieConfig()
+                .get(WorkflowAppService.SYSTEM_LIB_PATH));
+        Path libpath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX
+                + ShareLibService.dateFormat.format(time));
+        fs.mkdirs(libpath);
 
-            Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
-            PigActionExecutor ae = new PigActionExecutor();
-            Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
-            jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
-            ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+        Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
+        Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
+        Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
+        Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
+        fs.mkdirs(pigPath);
+        fs.mkdirs(ooziePath);
+        fs.mkdirs(pigPath1);
+        fs.mkdirs(pigPath2);
 
-            verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "pig-10.jar", "oozie_luncher.jar");
+        createFiles(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
+        createFiles(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
 
-        }
-        finally {
-            services.destroy();
-        }
+        String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+                + getNameNodeUri() + "</name-node>"
+                + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
+        Element eActionXml = XmlUtils.parseXml(actionXml);
+        XConfiguration protoConf = new XConfiguration();
+        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+        WorkflowJobBean wfj = new WorkflowJobBean();
+        protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+        wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
+
+        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+        PigActionExecutor ae = new PigActionExecutor();
+        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+        jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
+        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+
+        verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "pig-10.jar", "oozie_luncher.jar");
     }
 
     @Test
     public void testShareLibLoadFileMultipleFile() throws Exception {
         FileSystem fs = getFileSystem();
-        services = new Services();
         createTestShareLibMetaFile_multipleFile(fs);
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
-        try {
-            services.init();
-            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            assertNull(shareLibService.getShareLibJars("something_new"));
-            assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
-            fs.delete(new Path("shareLibPath/"), true);
-        }
-        finally {
-            services.destroy();
-        }
+        Configuration conf = getOozieConfig();
+        setShipLauncherInOozieConfig();
+        setShareLibMappingFileInOozieConfig(fs, conf);
+
+        services.init();
+        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+        assertNull(shareLibService.getShareLibJars("something_new"));
+        assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
+        fs.delete(new Path(SHARELIB_PATH), true);
+    }
+
+    private void setShareLibMappingFileInOozieConfig(FileSystem fs, Configuration conf) {
+        conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + TEST_HDFS_HOME + TEST_MAPPING_FILENAME);
     }
 
     @Test
     public void testMultipleLauncherCall() throws Exception {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
-        try {
-            FileSystem fs = getFileSystem();
-            Date time = new Date(System.currentTimeMillis());
-            Path basePath = new Path(services.get(ConfigurationService.class).getConf()
-                    .get(WorkflowAppService.SYSTEM_LIB_PATH));
-            Path libpath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX
-                    + ShareLibService.dateFormat.format(time));
-            fs.mkdirs(libpath);
-            Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
-            fs.mkdirs(ooziePath);
-            createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
-            services.init();
-            ShareLibService shareLibService = Services.get().get(ShareLibService.class);
-            List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
-            assertEquals(launcherPath.size(), 2);
-            launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
-            assertEquals(launcherPath.size(), 2);
-        }
-        finally {
-            services.destroy();
-        }
+        setShipLauncherInOozieConfig();
+        FileSystem fs = getFileSystem();
+        Date time = new Date(System.currentTimeMillis());
+        Path basePath = new Path(getOozieConfig()
+                .get(WorkflowAppService.SYSTEM_LIB_PATH));
+        Path libpath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX
+                + ShareLibService.dateFormat.format(time));
+        fs.mkdirs(libpath);
+        Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
+        fs.mkdirs(ooziePath);
+        createFiles(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
+        services.init();
+        ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+        List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
+        assertEquals(launcherPath.size(), 2);
+        launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
+        assertEquals(launcherPath.size(), 2);
     }
 
     @Test
     public void testMetafileSymlink() throws ServiceException, IOException {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        Configuration conf = getOozieConfig();
+        setShipLauncherInOozieConfig();
         services.init();
         FileSystem fs = getFileSystem();
         Properties prop = new Properties();
         try {
-
-            String testPath = "shareLibPath/";
-
-            Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
-            Path basePath1 = new Path(testPath + Path.SEPARATOR + "testPath1");
+            Path basePath = new Path(SHARELIB_PATH + "testPath");
+            Path basePath1 = new Path(SHARELIB_PATH + "testPath1");
             Path hive_site = new Path(basePath.toString() + Path.SEPARATOR + "hive_conf" + Path.SEPARATOR
                     + "hive-site.xml");
             Path hive_site1 = new Path(basePath.toString() + Path.SEPARATOR + "hive_conf" + Path.SEPARATOR
@@ -636,26 +568,26 @@ public class TestShareLibService extends XFsTestCase {
 
             fs.mkdirs(basePath);
 
-            createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
-            createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_1.jar");
+            createFiles(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
+            createFiles(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_1.jar");
 
-            createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_2.jar");
-            createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_3.jar");
-            createFile(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_4.jar");
+            createFiles(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_2.jar");
+            createFiles(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_3.jar");
+            createFiles(basePath1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig_4.jar");
 
-            createFile(hive_site.toString());
+            createFiles(hive_site.toString());
 
             FSUtils.createSymlink(fs, basePath, symlink, true);
             FSUtils.createSymlink(fs, hive_site, symlink_hive_site, true);
 
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + symlink.toString());
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".hive_conf", "/user/test/" + symlink_hive_site.toString()
+            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", TEST_HDFS_HOME + symlink.toString());
+            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".hive_conf", TEST_HDFS_HOME + symlink_hive_site.toString()
                     + "#hive-site.xml");
-            createTestShareLibMetaFile(fs, prop);
+            createTestShareLibMappingFile(fs, prop);
             assertEquals(FSUtils.isSymlink(fs, symlink), true);
 
-            conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
-            conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+            setShareLibMappingFileInOozieConfig(fs, conf);
+            setShipLauncherInOozieConfig();
             try {
                 ShareLibService shareLibService = Services.get().get(ShareLibService.class);
                 assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
@@ -667,28 +599,25 @@ public class TestShareLibService extends XFsTestCase {
                 assertEquals(shareLibService.getShareLibJars("pig").size(), 3);
             }
             finally {
-                fs.delete(new Path("shareLibPath/"), true);
-                fs.delete(new Path(metaFile), true);
-                fs.delete(new Path("/user/test/config.properties"), true);
-
                 fs.delete(symlink, true);
-                services.destroy();
             }
         }
         catch (IOException ex) {
             ex.printStackTrace();
         }
+        finally {
+            fs.delete(new Path(SHARELIB_PATH), true);
+            fs.delete(new Path(TEST_HDFS_MAPPING_FILE_PATH), true);
+        }
     }
 
     @Test
     public void testDuplicateJarsInDistributedCache() throws Exception {
-        services = new Services();
-        setSystemProps();
+
         FileSystem fs = getFileSystem();
-        Path basePath = new Path(services.get(ConfigurationService.class).getConf()
+        Path basePath = new Path(getOozieConfig()
                 .get(WorkflowAppService.SYSTEM_LIB_PATH));
-        Configuration conf = services.get(ConfigurationService.class).getConf();
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+        setShipLauncherInOozieConfig();
 
         // Use timedstamped directory if available
         Date time = new Date(System.currentTimeMillis());
@@ -696,24 +625,18 @@ public class TestShareLibService extends XFsTestCase {
 
         Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
         createDirs(fs, pigPath, new Path(pigPath, "temp"));
-        createFile(new Path(pigPath, "pig.jar"));
-        createFile(new Path(pigPath, "hive.jar"));
-        createFile(new Path(new Path(pigPath, "temp"), "pig.jar#pig.jar"));
-
-        try {
-
-            // DistributedCache should have only one pig jar
-            verifyFilesInDistributedCache(setUpPigJob(true), "pig.jar", "hive.jar", "MyOozie.jar", "MyPig.jar");
-            ShareLibService shareLibService = services.get(ShareLibService.class);
-            // sharelib service should have two jars
-            List<Path> shareLib = shareLibService.getShareLibJars("pig");
-            assertEquals(shareLib.size(), 3);
-            assertTrue(shareLib.toString().contains("pig.jar#pig.jar"));
-            assertTrue(shareLib.toString().contains("hive.jar"));
-        }
-        finally {
-            services.destroy();
-        }
+        createFiles(new Path(pigPath, "pig.jar"));
+        createFiles(new Path(pigPath, "hive.jar"));
+        createFiles(new Path(new Path(pigPath, "temp"), "pig.jar#pig.jar"));
+
+        // DistributedCache should have only one pig jar
+        verifyFilesInDistributedCache(setUpPigJob(true), "pig.jar", "hive.jar", "MyOozie.jar", "MyPig.jar");
+        ShareLibService shareLibService = services.get(ShareLibService.class);
+        // sharelib service should have two jars
+        List<Path> shareLib = shareLibService.getShareLibJars("pig");
+        assertEquals(shareLib.size(), 3);
+        assertTrue(shareLib.toString().contains("pig.jar#pig.jar"));
+        assertTrue(shareLib.toString().contains("hive.jar"));
     }
 
     private URI[] setUpPigJob(boolean useSystemSharelib) throws Exception {
@@ -736,66 +659,173 @@ public class TestShareLibService extends XFsTestCase {
         return DistributedCache.getCacheFiles(jobConf);
     }
 
-    private void createFile(String... filenames) throws IOException {
-        for (String filename : filenames) {
-            Path path = new Path(filename);
-            createFile(path);
+    private static void createFiles(FileSystem fs, Path... paths) throws IOException {
+        for (Path path : paths) {
+            FSDataOutputStream out = fs.create(path);
+            out.close();
         }
     }
 
-    private void createFile(Path... paths) throws IOException {
-        for (Path path : paths) {
-            FSDataOutputStream out = getFileSystem().create(path);
-            out.close();
+    private static void createFiles(FileSystem fs, String... filenames) throws IOException {
+        Path[] paths = new Path[filenames.length];
+        for (int i = 0; i != filenames.length; ++i) {
+            paths[i] = new Path(filenames[i]);
         }
+        createFiles(fs, paths);
     }
 
-    private void createTestShareLibMetaFile(FileSystem fs) {
-        Properties prop = new Properties();
+    private void createFiles(String... filenames) throws IOException {
+        createFiles(getFileSystem(), filenames);
+    }
+
+    private void createFiles(Path... paths) throws IOException {
+        createFiles(getFileSystem(), paths);
+    }
+
+    private void createShareLibMetaFileTestResources(final FileSystem fs, final String testUserHome)
+            throws IOException {
+        final String testPath = testUserHome + SHARELIB_PATH;
+
+        final Path basePath = new Path(testPath  + "testPath");
+        final Path somethingNew = new Path(testPath + "something_new");
+        final Path directJarDir = new Path(testPath + "directJarDir");
+        final String directJarPath = directJarDir.toString() + Path.SEPARATOR + "direct.jar";
+
+        final String[] testFiles = {
+                basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar" ,
+                somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR + "somethingNew.jar",
+                directJarDir.toString() + Path.SEPARATOR + "direct.jar"};
+
+        final Map<String, String> symlinks = new HashMap<>();
+        symlinks.put(testPath + Path.SEPARATOR + "linkDir" + Path.SEPARATOR + "targetOfLinkFile.xml",
+                testUserHome + "linkFile.xml");
+
+        final Properties mappingFileConfig = new Properties();
+        mappingFileConfig.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig",
+                TestShareLibMappingFileInput.getLocalizedShareLibPath(fs, basePath.toString()));
+        mappingFileConfig.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new",
+                TestShareLibMappingFileInput.getLocalizedShareLibPath(fs, somethingNew.toString()));
+        mappingFileConfig.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".directjar",
+                TestShareLibMappingFileInput.getLocalizedShareLibPath(fs, directJarPath.toString()));
+        String symlink = testUserHome + "linkFile.xml";
+        mappingFileConfig.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".linkFile",
+                TestShareLibMappingFileInput.getLocalizedShareLibPath(fs, symlink + "#targetOfLinkFile.xml"));
+
+        createTestShareLibMappingFile(fs, testUserHome, Arrays.asList(testFiles), symlinks, mappingFileConfig);
+    }
+
+    @Test
+    public void testLoadMappingFilesFromDFSandLocalFs() throws IOException, ServiceException {
+        final String testUserHome = Files.createTempDir().toString() + Path.SEPARATOR;
+        final String testPath = testUserHome + SHARELIB_PATH;
+        final Path basePath = new Path(testPath + "testPath");
+        final Path somethingNew = new Path(testPath + "something_new");
+        final Path directHdfsJarDir = new Path(TEST_HDFS_HOME + SHARELIB_PATH +  "directJarDir");
+        final String directHdfsJarPath = directHdfsJarDir.toString() + Path.SEPARATOR + "direct.jar";
+        final String directjarShareLibName = "directjar";
+
+        final FileSystem localFs = newLocalFileSystem();
+        final FileSystem hdFs = getFileSystem();
+        final TestShareLibMappingFileInput[] testShareLibMappingFileInputs = {
+                new TestShareLibMappingFileInput(localFs, "pig", basePath.toString() + Path.SEPARATOR + "pig"
+                        + Path.SEPARATOR + "pig.jar"),
+                new TestShareLibMappingFileInput(localFs, "something_new", somethingNew.toString()
+                        + Path.SEPARATOR + "something_new.jar"),
+                new TestShareLibMappingFileInput(hdFs, directjarShareLibName, directHdfsJarPath),
+                new TestShareLibMappingSymlinkInput(hdFs, "linkFile",
+                        TEST_HDFS_HOME + "symlinkTargetDir" + Path.SEPARATOR + "targetOfLinkFile.xml",
+                        TEST_HDFS_HOME + "linkFile.xml")
+        };
+
+        final Properties mappingFileConfig = new Properties();
+        for (final TestShareLibMappingFileInput shmfInput : testShareLibMappingFileInputs) {
+            shmfInput.materialize();
+            mappingFileConfig.put(shmfInput.sharelibNameWithMappingFilePrefix, shmfInput.getFullShareLibPathDir());
+        }
+
+        createTestShareLibMappingFile(testUserHome + TEST_MAPPING_FILENAME, localFs, mappingFileConfig);
+
+        setShipLauncherInOozieConfig();
+        final Configuration oozieConfig = getOozieConfig();
+        oozieConfig.set(ShareLibService.SHARELIB_MAPPING_FILE, localFs.getUri() + testUserHome + TEST_MAPPING_FILENAME);
 
         try {
+            services.init();
+            final ShareLibService shareLibService = Services.get().get(ShareLibService.class);
 
-            String testPath = "shareLibPath/";
-            String symlink = "/user/test/linkFile.xml";
+            for (final TestShareLibMappingFileInput sh : testShareLibMappingFileInputs) {
+                final String firstShareLibPath = shareLibService.getShareLibJars(sh.sharelibName).get(0).toString();
+                assertTrue(firstShareLibPath.endsWith(sh.baseName));
+            }
 
-            Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
-            Path somethingNew = new Path(testPath + Path.SEPARATOR + "something_new");
-            Path directJarDir = new Path(testPath + Path.SEPARATOR + "directJarDir");
-            Path linkDir = new Path(testPath + Path.SEPARATOR + "linkDir");
+            final List<Path> listOfPaths = shareLibService.getShareLibJars(directjarShareLibName);
+            for (final Path p : listOfPaths) {
+                assertTrue(p.toString().startsWith(HDFS_SCHEME_PREFIX));
+            }
 
-            fs.mkdirs(basePath);
-            fs.mkdirs(somethingNew);
-            fs.mkdirs(directJarDir);
-            fs.mkdirs(linkDir);
-
-            createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
-            createFile(somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR + "somethingNew.jar");
-            String directJarPath = directJarDir.toString() + Path.SEPARATOR + "direct.jar";
-            String symlinkTarget = linkDir.toString() + Path.SEPARATOR + "targetOfLinkFile.xml";
-            createFile(directJarPath);
-            createFile(symlinkTarget);
-            FSUtils.createSymlink(fs, new Path(symlinkTarget), new Path(symlink), true);
+        }
+        finally {
+            hdFs.delete(new Path(SHARELIB_PATH), true);
+            hdFs.delete(new Path("linkFile.xml"), true);
+            localFs.delete(new Path(testUserHome), true);
+        }
+    }
 
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString());
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".something_new", "/user/test/" + somethingNew.toString());
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".directjar",
-                    "/user/test/" + directJarPath.toString());
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".linkFile", symlink + "#targetOfLinkFile.xml");
+    private FileSystem newLocalFileSystem() throws IOException {
+        final Configuration emptyConfig = new Configuration(false);
+        return LocalFileSystem.get(emptyConfig);
+    }
+
+    private void setShipLauncherInOozieConfig() {
+        Configuration oozieConfig = getOozieConfig();
+        oozieConfig.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
+    }
+
+    private Configuration getOozieConfig() {
+        return services.get(ConfigurationService.class).getConf();
+    }
 
-            createTestShareLibMetaFile(fs, prop);
+    private void createTestShareLibMappingFile(final FileSystem fs,
+                                               final String testUserHome,
+                                               final Iterable<String> testFiles,
+                                               final Map<String, String> testSymlinks,
+                                               final Properties mappingConf) throws IOException {
+        createTestShareLibDirsAndFiles(fs, testFiles);
+        createTestShareLibDirsAndSymlinks(fs, testSymlinks);
+        createTestShareLibMappingFile(testUserHome + TEST_MAPPING_FILENAME, fs, mappingConf);
+    }
 
+    private void createTestShareLibDirsAndFiles(final FileSystem fs, final Iterable<String> testFiles)
+            throws IOException {
+        for (final String f : testFiles) {
+            createFileWithDirectoryPath(fs, f);
         }
-        catch (IOException ex) {
-            ex.printStackTrace();
+    }
+
+    static void createFileWithDirectoryPath(final FileSystem fs, final String f) throws IOException {
+        if (!fs.exists(new Path(f))) {
+            String dirName = f.substring(0, f.toString().lastIndexOf(Path.SEPARATOR));
+            fs.mkdirs(new Path(dirName));
         }
+        createFiles(fs, f);
     }
 
-    private void createTestShareLibMetaFile(FileSystem fs, Properties prop) {
-        try {
-            FSDataOutputStream out = fs.create(new Path(metaFile));
-            prop.store(out, null);
-            out.close();
+    void createTestShareLibDirsAndSymlinks(final FileSystem fs, final Map<String, String> symlinks) throws IOException {
+        createTestShareLibDirsAndFiles(fs, symlinks.keySet()); // create symlink targets
+        for (Map.Entry<String, String> symlinkEntry : symlinks.entrySet())  {
+            String symlinkTarget = symlinkEntry.getKey();
+            String symlink = symlinkEntry.getValue();
+            FSUtils.createSymlink(fs, new Path(symlinkTarget), new Path(symlink), true);
+        }
+    }
+
+    private void createTestShareLibMappingFile(final FileSystem fs, final Properties prop) {
+        createTestShareLibMappingFile(TEST_HDFS_MAPPING_FILE_PATH, fs, prop);
+    }
 
+    private void createTestShareLibMappingFile(final String metaFile, final FileSystem fs, final Properties prop) {
+        try (final FSDataOutputStream out = fs.create(new Path(metaFile))) {
+            prop.store(out, null);
         }
         catch (IOException ex) {
             ex.printStackTrace();
@@ -803,25 +833,22 @@ public class TestShareLibService extends XFsTestCase {
     }
 
     public void createTestShareLibMetaFile_multipleFile(FileSystem fs) {
-
         Properties prop = new Properties();
-        try {
-
-            String testPath = "shareLibPath/";
 
-            Path basePath = new Path(testPath + Path.SEPARATOR + "testPath");
-            Path somethingNew = new Path(testPath + Path.SEPARATOR + "something_new");
+        try {
+            Path basePath = new Path(SHARELIB_PATH + "testPath");
+            Path somethingNew = new Path(SHARELIB_PATH + "something_new");
             fs.mkdirs(basePath);
             fs.mkdirs(somethingNew);
 
-            createFile(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
-            createFile(somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR + "somethingNew.jar");
+            createFiles(basePath.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
+            createFiles(somethingNew.toString() + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR + "somethingNew.jar");
 
-            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", "/user/test/" + basePath.toString()
-                    + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar," + "/user/test/" + somethingNew.toString()
+            prop.put(ShareLibService.SHARE_LIB_CONF_PREFIX + ".pig", TEST_HDFS_HOME + basePath.toString()
+                    + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar," + TEST_HDFS_HOME + somethingNew.toString()
                     + Path.SEPARATOR + "somethingNew" + Path.SEPARATOR + "somethingNew.jar");
 
-            FSDataOutputStream out = fs.create(new Path(metaFile));
+            FSDataOutputStream out = fs.create(new Path(TEST_HDFS_MAPPING_FILE_PATH));
 
             prop.store(out, null);
             out.close();
@@ -840,9 +867,7 @@ public class TestShareLibService extends XFsTestCase {
 
     @Test
     public void testConfFileAddedToActionConf() throws Exception {
-
         try {
-
             XConfiguration protoConf = new XConfiguration();
             protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
             WorkflowJobBean wfj = new WorkflowJobBean();
@@ -881,7 +906,7 @@ public class TestShareLibService extends XFsTestCase {
             jobConf = ae.createBaseHadoopConf(context, eActionXml);
             Properties prop = new Properties();
             actionConf = ae.createBaseHadoopConf(context, eActionXml);
-            prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
+            prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
             setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
             jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
             ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
@@ -892,7 +917,7 @@ public class TestShareLibService extends XFsTestCase {
             prop = new Properties();
             jobConf = ae.createBaseHadoopConf(context, eActionXml);
             actionConf = ae.createBaseHadoopConf(context, eActionXml);
-            prop.put("oozie.hive_conf", "hdfs:///user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
+            prop.put("oozie.hive_conf", "hdfs://" + TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
             setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
             jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
             ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
@@ -905,7 +930,7 @@ public class TestShareLibService extends XFsTestCase {
             prop = new Properties();
             jobConf = ae.createBaseHadoopConf(context, eActionXml);
             actionConf = ae.createBaseHadoopConf(context, eActionXml);
-            prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml");
+            prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml");
             setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
             jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
             ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
@@ -919,7 +944,7 @@ public class TestShareLibService extends XFsTestCase {
             prop = new Properties();
             jobConf = ae.createBaseHadoopConf(context, eActionXml);
             actionConf = ae.createBaseHadoopConf(context, eActionXml);
-            prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
+            prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
             setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
             jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
             ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
@@ -927,11 +952,9 @@ public class TestShareLibService extends XFsTestCase {
             cacheFiles = DistributedCache.getCacheFiles(actionConf);
             cacheFilesStr = Arrays.toString(cacheFiles);
             assertFalse(cacheFilesStr.contains("hive-site.xml"));
-
         }
         finally {
-            getFileSystem().delete(new Path(sharelibPath), true);
-            services.destroy();
+            getFileSystem().delete(new Path(SHARELIB_PATH), true);
         }
     }
 
@@ -940,7 +963,7 @@ public class TestShareLibService extends XFsTestCase {
         try {
 
             Properties prop = new Properties();
-            prop.put("oozie.hive_conf", "/user/test/" + sharelibPath + "/hive-site.xml#hive-site.xml");
+            prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
             setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
 
             String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
@@ -974,41 +997,40 @@ public class TestShareLibService extends XFsTestCase {
 
         }
         finally {
-            getFileSystem().delete(new Path(sharelibPath), true);
-            services.destroy();
+            getFileSystem().delete(new Path(SHARELIB_PATH), true);
         }
     }
 
     private void setupSharelibConf(final String file, final String tag) throws ServiceException, IOException {
         Properties prop = new Properties();
-        prop.put(tag, "/user/test/" + sharelibPath);
+        prop.put(tag, TEST_HDFS_HOME + SHARELIB_PATH);
         setupSharelibConf(file, tag, prop);
-
     }
 
     private void setupSharelibConf(final String file, final String tag, Properties prop) throws IOException,
             ServiceException {
-        services = new Services();
-        setSystemProps();
-        Configuration conf = services.get(ConfigurationService.class).getConf();
+        setupSharelibConf(getFileSystem(), file, tag, prop);
+    }
 
-        conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
-        conf.set(ShareLibService.SHARELIB_MAPPING_FILE, getFileSystem().getUri() + "/user/test/config.properties");
+    private void setupSharelibConf(final FileSystem fs, final String file, final String tag, Properties prop)
+            throws IOException, ServiceException {
+        Configuration conf = getOozieConfig();
+        setShipLauncherInOozieConfig();
+        setShareLibMappingFileInOozieConfig(fs, conf);
 
         XConfiguration hiveConf = new XConfiguration();
         hiveConf.set(tag + "-sharelib-test", "test");
-        createDirs(getFileSystem(), new Path(sharelibPath));
-        FSDataOutputStream out = getFileSystem().create(new Path(sharelibPath, file));
+        createDirs(getFileSystem(), new Path(SHARELIB_PATH));
+        FSDataOutputStream out = getFileSystem().create(new Path(SHARELIB_PATH, file), true);
         PrintWriter bufOut = new PrintWriter(out);
         bufOut.write(hiveConf.toXmlString(false));
         bufOut.close();
-        createTestShareLibMetaFile(getFileSystem(), prop);
+        createTestShareLibMappingFile(getFileSystem(), prop);
 
         services.init();
     }
 
     private void verifyFilesInDistributedCache(URI[] cacheFiles, String... files) {
-
         String cacheFilesStr = Arrays.toString(cacheFiles);
         // Hadoop 2 has the following jars too: MRAppJar.jar and hadoop-mapreduce-client-jobclient-
         assertEquals(cacheFiles.length, files.length + 2);
@@ -1017,9 +1039,6 @@ public class TestShareLibService extends XFsTestCase {
 
         for (String file : files) {
             assertTrue(cacheFilesStr.contains(file));
-
         }
-
     }
-
 }

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/docs/src/site/twiki/AG_Install.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki
index 3509266..dc8ee0f 100644
--- a/docs/src/site/twiki/AG_Install.twiki
+++ b/docs/src/site/twiki/AG_Install.twiki
@@ -1120,8 +1120,8 @@ time-stamp directory. While starting, server also purges sharelib directory whic
 (defined as oozie.service.ShareLibService.temp.sharelib.retention.days and 7 days is default).
 
 Sharelib mapping file can be also configured. Configured file is a key value mapping, where key will be the sharelib name for the
-action and value is a comma separated list of DFS directories or jar files.
- This can be configured in oozie-site.xml as :
+action and value is a comma separated list of DFS or local filesystem directories or jar files. Local filesystem refers to the local
+filesystem of the node where the Oozie launcher is running. This can be configured in oozie-site.xml as :
  <verbatim>
   <!-- OOZIE -->
     <property>
@@ -1130,16 +1130,40 @@ action and value is a comma separated list of DFS directories or jar files.
         <description>
             Sharelib mapping files contains list of key=value,
             where key will be the sharelib name for the action and value is a comma separated list of
-            DFS directories or jar files.
+            DFS or local filesystem directories or jar files.
             Example.
             oozie.pig_10=hdfs:///share/lib/pig/pig-0.10.1/lib/
             oozie.pig=hdfs:///share/lib/pig/pig-0.11.1/lib/
             oozie.distcp=hdfs:///share/lib/hadoop-2.2.0/share/hadoop/tools/lib/hadoop-distcp-2.2.0.jar
             oozie.spark=hdfs:///share/lib/spark/lib/,hdfs:///share/lib/spark/python/lib/pyspark.zip,hdfs:///share/lib/spark/python/lib/py4j-0-9-src.zip
+            oozie.hive=file:///usr/local/oozie/share/lib/hive/
         </description>
     </property>
  </verbatim>
 
+Example mapping file with local filesystem resources:
+
+ <verbatim>
+    <property>
+        <name>oozie.service.ShareLibService.mapping.file</name>
+        <value>
+            oozie.distcp=file:///usr/local/oozie/share/lib/distcp
+            oozie.hcatalog=file:///usr/local/oozie/share/lib/hcatalog
+            oozie.hive=file:///usr/local/oozie/share/lib/hive
+            oozie.hive2=file:///usr/local/oozie/share/lib/hive2
+            oozie.mapreduce-streaming=file:///usr/local/oozie/share/lib/mapreduce-streaming
+            oozie.oozie=file://usr/local/oozie/share/lib/oozie
+            oozie.pig=file:///usr/local/oozie/share/lib/pig
+            oozie.spark=file:///usr/local/oozie/share/lib/spark
+            oozie.sqoop=file:///usr/localoozie/share/lib/sqoop
+        </value>
+    </property>
+  </verbatim>
+
+If you are using local filesystem resources in the mapping file, make sure corresponding jars are already deployed to
+all the nodes where Oozie launcher jobs will be executed, and the files are readable by the launchers. To do this, you
+can extract Oozie sharelib TAR.GZ file in the directory of your choice on the nodes, and set permission of the files.
+
 Oozie sharelib TAR.GZ file bundled with the distribution does not contain pyspark and py4j zip files since they vary
 with Apache Spark version. Therefore, to run pySpark using Spark Action, user need to specify pyspark and py4j zip
 files. These files can be added either to workflow's lib/ directory, to the sharelib or in sharelib mapping file.

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index ccb845c..05f53bc 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 5.0.0 release (trunk - unreleased)
 
+OOZIE-2601 Ability to use local paths for the sharelib
 OOZIE-2987 Coord action missing dependencies should show URI template with unresolved dependencies
 OOZIE-2004 Improve Oozie version info output (Artem Ervits via gezapeti)
 OOZIE-2854 Oozie should handle transient database problems (andras.piros via gezapeti)


[2/2] oozie git commit: OOZIE-2601 Ability to use local paths for the sharelib

Posted by as...@apache.org.
OOZIE-2601 Ability to use local paths for the sharelib


Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/25a8b99d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/25a8b99d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/25a8b99d

Branch: refs/heads/master
Commit: 25a8b99d5197c4e18acf0fd332c4396450d3d551
Parents: 18694b6
Author: Attila Sasvari <as...@cloudera.com>
Authored: Thu Jul 20 13:20:47 2017 +0200
Committer: Attila Sasvari <as...@apache.org>
Committed: Thu Jul 20 17:18:20 2017 +0200

----------------------------------------------------------------------
 .../oozie/action/hadoop/JavaActionExecutor.java | 109 ++-
 .../apache/oozie/service/ShareLibService.java   |  75 +-
 .../org/apache/oozie/util/ClasspathUtils.java   |  27 +
 .../java/org/apache/oozie/util/FSUtils.java     |  14 +
 core/src/main/resources/oozie-default.xml       |   3 +-
 .../service/TestShareLibMappingFileInput.java   |  88 ++
 .../oozie/service/TestShareLibService.java      | 873 ++++++++++---------
 docs/src/site/twiki/AG_Install.twiki            |  30 +-
 release-log.txt                                 |   1 +
 9 files changed, 724 insertions(+), 496 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 89bac95..23e1f69 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -18,44 +18,24 @@
 
 package org.apache.oozie.action.hadoop;
 
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.ConnectException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.commons.io.IOUtils;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Closeables;
 import com.google.common.primitives.Ints;
+import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.TaskLog;
 import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -95,6 +75,7 @@ import org.apache.oozie.service.WorkflowAppService;
 import org.apache.oozie.util.ClasspathUtils;
 import org.apache.oozie.util.ELEvaluationException;
 import org.apache.oozie.util.ELEvaluator;
+import org.apache.oozie.util.FSUtils;
 import org.apache.oozie.util.JobUtils;
 import org.apache.oozie.util.LogUtils;
 import org.apache.oozie.util.PropertiesUtils;
@@ -105,8 +86,28 @@ import org.jdom.Element;
 import org.jdom.JDOMException;
 import org.jdom.Namespace;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.Closeables;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.ConnectException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Set;
 
 
 public class JavaActionExecutor extends ActionExecutor {
@@ -463,8 +464,11 @@ public class JavaActionExecutor extends ActionExecutor {
                 else if (fileName.endsWith(".jar")) { // .jar files
                     if (!fileName.contains("#")) {
                         String user = conf.get("user.name");
-                        Path pathToAdd = new Path(uri.normalize());
-                        Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, pathToAdd, conf);
+
+                        if (FSUtils.isNotLocalFile(fileName)) {
+                            Path pathToAdd = new Path(uri.normalize());
+                            Services.get().get(HadoopAccessorService.class).addFileToClassPath(user, pathToAdd, conf);
+                        }
                     }
                     else {
                         DistributedCache.addCacheFile(uri.normalize(), conf);
@@ -585,9 +589,7 @@ public class JavaActionExecutor extends ActionExecutor {
                         }
                     }
                 }
-                for (Path libPath : sharelibList) {
-                    addToCache(conf, libPath, libPath.toUri().getPath(), false);
-                }
+                addLibPathsToCache(conf, sharelibList);
             }
             catch (URISyntaxException ex) {
                 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "Error configuring sharelib",
@@ -610,18 +612,8 @@ public class JavaActionExecutor extends ActionExecutor {
                     throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "EJ001",
                             "Could not locate Oozie sharelib");
                 }
-                FileSystem fs = listOfPaths.get(0).getFileSystem(conf);
-                for (Path actionLibPath : listOfPaths) {
-                    JobUtils.addFileToClassPath(actionLibPath, conf, fs);
-                    DistributedCache.createSymlink(conf);
-                }
-                listOfPaths = shareLibService.getSystemLibJars(getType());
-                if (!listOfPaths.isEmpty()) {
-                    for (Path actionLibPath : listOfPaths) {
-                        JobUtils.addFileToClassPath(actionLibPath, conf, fs);
-                        DistributedCache.createSymlink(conf);
-                    }
-                }
+                addLibPathsToClassPath(conf, listOfPaths);
+                addLibPathsToClassPath(conf, shareLibService.getSystemLibJars(getType()));
             }
             catch (IOException ex) {
                 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "It should never happen",
@@ -630,6 +622,37 @@ public class JavaActionExecutor extends ActionExecutor {
         }
     }
 
+    private void addLibPathsToClassPath(Configuration conf, List<Path> listOfPaths)
+            throws IOException, ActionExecutorException {
+        addLibPathsToClassPathOrCache(conf, listOfPaths, false);
+    }
+
+    private void addLibPathsToCache(Configuration conf, Set<Path> sharelibList)
+            throws IOException, ActionExecutorException {
+        addLibPathsToClassPathOrCache(conf, sharelibList, true);
+    }
+
+
+    private void addLibPathsToClassPathOrCache(Configuration conf, Collection<Path> sharelibList, boolean isAddToCache)
+            throws IOException, ActionExecutorException {
+
+        for (Path libPath : sharelibList) {
+            if (FSUtils.isLocalFile(libPath.toString())) {
+                conf = ClasspathUtils.addToClasspathFromLocalShareLib(conf, libPath);
+            }
+            else {
+                if (isAddToCache) {
+                    addToCache(conf, libPath, libPath.toUri().getPath(), false);
+                }
+                else {
+                    FileSystem fs = libPath.getFileSystem(conf);
+                    JobUtils.addFileToClassPath(libPath, conf, fs);
+                    DistributedCache.createSymlink(conf);
+                }
+            }
+        }
+    }
+
     protected void addActionLibs(Path appPath, Configuration conf) throws ActionExecutorException {
         String[] actionLibsStrArr = conf.getStrings("oozie.launcher.oozie.libpath");
         if (actionLibsStrArr != null) {

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 82a0aff..8986a49 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -45,6 +45,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -60,6 +61,8 @@ import org.apache.oozie.util.XConfiguration;
 import org.apache.oozie.util.XLog;
 import org.jdom.JDOMException;
 
+import static org.apache.oozie.util.FSUtils.isLocalFile;
+
 public class ShareLibService implements Service, Instrumentable {
 
     public static final String LAUNCHERJAR_LIB_RETENTION = CONF_PREFIX + "ShareLibService.temp.sharelib.retention.days";
@@ -108,6 +111,7 @@ public class ShareLibService implements Service, Instrumentable {
     private String sharelibDirOld;
 
     FileSystem fs;
+    FileSystem localFs;
 
     final long retentionTime = 1000 * 60 * 60 * 24 * ConfigurationService.getInt(LAUNCHERJAR_LIB_RETENTION);
 
@@ -121,7 +125,9 @@ public class ShareLibService implements Service, Instrumentable {
         HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
         URI uri = launcherlibPath.toUri();
         try {
+
             fs = FileSystem.get(has.createConfiguration(uri.getAuthority()));
+            localFs = LocalFileSystem.get(new Configuration(false));
             //cache action key sharelib conf list
             cacheActionKeySharelibConfList();
             updateLauncherLib();
@@ -350,24 +356,26 @@ public class ShareLibService implements Service, Instrumentable {
         return shareLibMap.get(shareLibKey);
     }
 
-    private void checkSymlink(String shareLibKey) throws IOException {
+    private void checkSymlink(final String shareLibKey) throws IOException {
         if (symlinkMapping.get(shareLibKey) == null || symlinkMapping.get(shareLibKey).isEmpty()) {
             return;
         }
 
-        for (Path path : symlinkMapping.get(shareLibKey).keySet()) {
-            if (!symlinkMapping.get(shareLibKey).get(path).equals(FSUtils.getSymLinkTarget(fs, path))) {
+        for (final Path symlinkPath : symlinkMapping.get(shareLibKey).keySet()) {
+            final FileSystem fileSystem = getHostFileSystem(symlinkPath);
+            final Path symLinkTarget = FSUtils.getSymLinkTarget(fileSystem, symlinkPath);
+            final boolean symlinkIsNotTarget = !getSymlinkSharelibPath(shareLibKey, symlinkPath).equals(symLinkTarget);
+            if (symlinkIsNotTarget) {
                 synchronized (ShareLibService.class) {
-                    Map<String, List<Path>> tmpShareLibMap = new HashMap<String, List<Path>>(shareLibMap);
+                    final Map<String, List<Path>> tmpShareLibMap = new HashMap<String, List<Path>>(shareLibMap);
 
-                    Map<String, Map<Path, Configuration>> tmpShareLibConfigMap = new HashMap<String, Map<Path, Configuration>>(
-                            shareLibConfigMap);
+                    final Map<String, Map<Path, Configuration>> tmpShareLibConfigMap = new HashMap<>(shareLibConfigMap);
 
-                    Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>(
+                    final Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>(
                             symlinkMapping);
 
                     LOG.info(MessageFormat.format("Symlink target for [{0}] has changed, was [{1}], now [{2}]",
-                            shareLibKey, path, FSUtils.getSymLinkTarget(fs, path)));
+                            shareLibKey, symlinkPath, symLinkTarget));
                     loadShareLibMetaFile(tmpShareLibMap, tmpSymlinkMapping, tmpShareLibConfigMap, sharelibMappingFile,
                             shareLibKey);
                     shareLibMap = tmpShareLibMap;
@@ -375,10 +383,27 @@ public class ShareLibService implements Service, Instrumentable {
                     shareLibConfigMap = tmpShareLibConfigMap;
                     return;
                 }
-
             }
         }
+    }
 
+    private Path getSymlinkSharelibPath(String shareLibKey, Path path) {
+        return symlinkMapping.get(shareLibKey).get(path);
+    }
+
+    private FileSystem getHostFileSystem(String pathStr) {
+        FileSystem fileSystem;
+        if (isLocalFile(pathStr)) {
+            fileSystem = localFs;
+        }
+        else {
+            fileSystem = fs;
+        }
+        return fileSystem;
+    }
+
+    private FileSystem getHostFileSystem(Path path) {
+        return getHostFileSystem(path.toString());
     }
 
     /**
@@ -534,9 +559,13 @@ public class ShareLibService implements Service, Instrumentable {
         Map<String, Map<Path, Path>> tmpSymlinkMapping = new HashMap<String, Map<Path, Path>>();
         Map<String, Map<Path, Configuration>> tmpShareLibConfigMap = new HashMap<String, Map<Path, Configuration>>();
 
-        if (!StringUtils.isEmpty(sharelibMappingFile.trim())) {
+        String trimmedSharelibMappingFile = sharelibMappingFile.trim();
+        if (!StringUtils.isEmpty(trimmedSharelibMappingFile)) {
+            FileSystem fileSystem = getHostFileSystem(trimmedSharelibMappingFile);
+
             String sharelibMetaFileNewTimeStamp = JsonUtils.formatDateRfc822(
-                    new Date(fs.getFileStatus(new Path(sharelibMappingFile)).getModificationTime()), "GMT");
+                    new Date(fileSystem.getFileStatus(new Path(sharelibMappingFile)).getModificationTime()), "GMT");
+
             loadShareLibMetaFile(tempShareLibMap, tmpSymlinkMapping, tmpShareLibConfigMap, sharelibMappingFile, null);
             status.put("sharelibMetaFile", sharelibMappingFile);
             status.put("sharelibMetaFileNewTimeStamp", sharelibMetaFileNewTimeStamp);
@@ -612,8 +641,8 @@ public class ShareLibService implements Service, Instrumentable {
             throws IOException {
 
         Path shareFileMappingPath = new Path(sharelibFileMapping);
-        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
-        FileSystem filesystem = FileSystem.get(has.createConfiguration(shareFileMappingPath.toUri().getAuthority()));
+        FileSystem filesystem = getHostFileSystem(shareFileMappingPath);
+
         Properties prop = new Properties();
         prop.load(filesystem.open(new Path(sharelibFileMapping)));
 
@@ -634,11 +663,13 @@ public class ShareLibService implements Service, Instrumentable {
         List<Path> listOfPaths = new ArrayList<Path>();
         Map<Path, Path> symlinkMappingforAction = new HashMap<Path, Path>();
 
-        for (String dfsPath : pathList) {
-            Path path = new Path(dfsPath);
-            getPathRecursively(fs, new Path(dfsPath), listOfPaths, shareLibKey, shareLibConfigMap);
-            if (FSUtils.isSymlink(fs, path)) {
-                symlinkMappingforAction.put(path, FSUtils.getSymLinkTarget(fs, path));
+        for (String pathStr : pathList) {
+            Path path = new Path(pathStr);
+            final FileSystem fileSystem = getHostFileSystem(pathStr);
+
+            getPathRecursively(fileSystem, path, listOfPaths, shareLibKey, shareLibConfigMap);
+            if (FSUtils.isSymlink(fileSystem, path)) {
+                symlinkMappingforAction.put(path, FSUtils.getSymLinkTarget(fileSystem, path));
             }
         }
 
@@ -825,21 +856,21 @@ public class ShareLibService implements Service, Instrumentable {
     /**
      * Cache XML conf file
      *
-     * @param hdfsPath the hdfs path
+     * @param propertyFilePath the path of the property file
      * @param shareLibKey the share lib key
      * @throws IOException Signals that an I/O exception has occurred.
      * @throws JDOMException
      */
-    private void cachePropertyFile(Path qualifiedHdfsPath, Path hdfsPath, String shareLibKey,
+    private void cachePropertyFile(Path qualifiedHdfsPath, Path propertyFilePath, String shareLibKey,
             Map<String, Map<Path, Configuration>> shareLibConfigMap) throws IOException, JDOMException {
         Map<Path, Configuration> confMap = shareLibConfigMap.get(shareLibKey);
         if (confMap == null) {
             confMap = new HashMap<Path, Configuration>();
             shareLibConfigMap.put(shareLibKey, confMap);
         }
-        Configuration xmlConf = new XConfiguration(fs.open(hdfsPath));
+        FileSystem fileSystem = getHostFileSystem(propertyFilePath);
+        Configuration xmlConf = new XConfiguration(fileSystem.open(propertyFilePath));
         confMap.put(qualifiedHdfsPath, xmlConf);
-
     }
 
     private void cacheActionKeySharelibConfList() {

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
index 5833607..bfe2f3e 100644
--- a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Arrays;
@@ -37,6 +38,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+
 public class ClasspathUtils {
     private static boolean usingMiniYarnCluster = false;
     private static final List<String> CLASSPATH_ENTRIES = Arrays.asList(
@@ -139,4 +141,29 @@ public class ClasspathUtils {
             }
         }
     }
+
+    public static Configuration addToClasspathFromLocalShareLib(Configuration conf, Path libPath) {
+        if (conf == null) {
+            conf = new Configuration(false);
+        }
+        final String pathStr = normalizedLocalFsPath(libPath);
+
+        String appClassPath = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
+
+        if (org.apache.commons.lang.StringUtils.isEmpty(appClassPath)) {
+            addPathToYarnClasspathInConfig(conf, pathStr, StringUtils.join(File.pathSeparator,
+                    YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
+        } else {
+            addPathToYarnClasspathInConfig(conf, pathStr, appClassPath);
+        }
+        return conf;
+    }
+
+    private static void addPathToYarnClasspathInConfig(Configuration conf, String pathStr, String appClassPath) {
+        conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, appClassPath + File.pathSeparator + pathStr);
+    }
+
+    private static String normalizedLocalFsPath(Path libPath) {
+        return org.apache.commons.lang.StringUtils.replace(libPath.toString(), FSUtils.FILE_SCHEME_PREFIX, "");
+    }
 }

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/main/java/org/apache/oozie/util/FSUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/FSUtils.java b/core/src/main/java/org/apache/oozie/util/FSUtils.java
index 6d73fc7..c25d931 100644
--- a/core/src/main/java/org/apache/oozie/util/FSUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/FSUtils.java
@@ -25,6 +25,8 @@ import java.io.IOException;
 import java.net.URI;
 
 public final class FSUtils {
+    public static final String FILE_SCHEME_PREFIX = "file:";
+
     public static Path getSymLinkTarget(FileSystem fs, Path p) throws IOException {
         try {
             //getSymlink doesn't work with fragment name, need to remove fragment before calling getSymlink
@@ -50,4 +52,16 @@ public final class FSUtils {
     public static void createSymlink(FileSystem fs, Path target, Path link, boolean createParent) throws IOException {
         fs.createSymlink(target, link, createParent);
     }
+
+    public static boolean isLocalFile(String fileName) {
+        return fileName.startsWith(FILE_SCHEME_PREFIX + "/");
+    }
+
+    public static boolean isLocalFile(Path filePath) {
+        return isLocalFile(filePath.toString());
+    }
+
+    public static boolean isNotLocalFile(String fileName) {
+       return !isLocalFile(fileName);
+    }
 }

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 832bbe1..b59a2e6 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -2694,11 +2694,12 @@ will be the requeue interval for the actions which are waiting for a long time w
         <description>
             Sharelib mapping files contains list of key=value,
             where key will be the sharelib name for the action and value is a comma separated list of
-            DFS directories or jar files.
+            DFS or local filesystem directories or jar files.
             Example.
             oozie.pig_10=hdfs:///share/lib/pig/pig-0.10.1/lib/
             oozie.pig=hdfs:///share/lib/pig/pig-0.11.1/lib/
             oozie.distcp=hdfs:///share/lib/hadoop-2.2.0/share/hadoop/tools/lib/hadoop-distcp-2.2.0.jar
+            oozie.hive=file:///usr/local/oozie/share/lib/hive/
         </description>
 
     </property>

http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/test/java/org/apache/oozie/service/TestShareLibMappingFileInput.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibMappingFileInput.java b/core/src/test/java/org/apache/oozie/service/TestShareLibMappingFileInput.java
new file mode 100644
index 0000000..92fec9b
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibMappingFileInput.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.service;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.util.FSUtils;
+
+import java.io.IOException;
+
+public class TestShareLibMappingFileInput {
+    final FileSystem fs;
+    final String sharelibNameWithMappingFilePrefix;
+    final String sharelibName;
+    final String sharelibPath;
+    final String baseName;
+
+    public TestShareLibMappingFileInput(final FileSystem fs,
+                                        final String sharelibMappingActionName,
+                                        final String sharelibPath) {
+        this.fs = fs;
+        this.sharelibName = sharelibMappingActionName;
+        this.sharelibNameWithMappingFilePrefix = ShareLibService.SHARE_LIB_CONF_PREFIX + "."
+                + sharelibMappingActionName;
+        this.sharelibPath = sharelibPath;
+        this.baseName = sharelibPath.substring(sharelibPath.lastIndexOf(Path.SEPARATOR) + 1);
+    }
+
+    public void materialize() throws IOException {
+        TestShareLibService.createFileWithDirectoryPath(fs, sharelibPath);
+    }
+
+    String getFullShareLibPathDir() {
+        String fullShareLibPathDir = getLocalizedShareLibPath(fs, sharelibPath);
+        return fullShareLibPathDir.substring(0, fullShareLibPathDir.lastIndexOf(Path.SEPARATOR));
+    }
+
+    static String getLocalizedShareLibPath(FileSystem fs, String path) {
+        return (fs instanceof LocalFileSystem)
+                ? FSUtils.FILE_SCHEME_PREFIX + "//" + path
+                : path;
+    }
+}
+
+class TestShareLibMappingSymlinkInput extends TestShareLibMappingFileInput {
+    final String symlinkPath;
+
+    public TestShareLibMappingSymlinkInput(final FileSystem fs,
+                                           final String sharelibActionName,
+                                           final String sharelibPath,
+                                           final String symlinkPath) {
+        super(fs, sharelibActionName, sharelibPath);
+        this.symlinkPath = symlinkPath;
+    }
+
+    @Override
+    public void materialize() throws IOException {
+            super.materialize();
+            FSUtils.createSymlink(fs, new Path(sharelibPath), new Path(symlinkPath), true);
+    }
+
+    @Override
+    String getFullShareLibPathDir() {
+        int lastIndexOfPathSeparator = sharelibPath.lastIndexOf(Path.SEPARATOR);
+        String symlinkTargetFileName = (lastIndexOfPathSeparator != -1)
+                ? sharelibPath.substring(lastIndexOfPathSeparator + 1)
+                : sharelibPath;
+        String fullShareLibPathDir = getLocalizedShareLibPath(fs, symlinkPath);
+        return fullShareLibPathDir + "#" + symlinkTargetFileName;
+    }
+}