You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2014/05/23 02:25:59 UTC
git commit: OOZIE-1783 Sharelib purging only occurs at Oozie startup
(rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 65e2109df -> b9c3e0b0c
OOZIE-1783 Sharelib purging only occurs at Oozie startup (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/b9c3e0b0
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/b9c3e0b0
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/b9c3e0b0
Branch: refs/heads/master
Commit: b9c3e0b0c9a62094d2a53034de53d6b4cc62b7c2
Parents: 65e2109
Author: Robert Kanter <rk...@cloudera.com>
Authored: Thu May 22 17:19:47 2014 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Thu May 22 17:19:47 2014 -0700
----------------------------------------------------------------------
.../apache/oozie/service/ShareLibService.java | 36 +-
core/src/main/resources/oozie-default.xml | 8 +
.../oozie/service/TestShareLibService.java | 401 ++++++++++---------
release-log.txt | 1 +
.../oozie/tools/TestOozieSharelibCLI.java | 7 +-
5 files changed, 254 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/core/src/main/java/org/apache/oozie/service/ShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
index 353b382..320af8b 100644
--- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java
+++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java
@@ -63,6 +63,8 @@ public class ShareLibService implements Service, Instrumentable {
public static final String SHIP_LAUNCHER_JAR = "oozie.action.ship.launcher.jar";
+ public static final String PURGE_INTERVAL = CONF_PREFIX + "ShareLibService.purge.interval";
+
private static final String PERMISSION_STRING = "-rwxr-xr-x";
public static final String LAUNCHER_PREFIX = "launcher_";
@@ -114,18 +116,25 @@ public class ShareLibService implements Service, Instrumentable {
+ "'oozie admin' CLI command to update the sharelib", ioe);
LOG.error(se);
}
- try {
- //Only one server should purge sharelib
- if (Services.get().get(JobsConcurrencyService.class).isFirstServer()) {
- final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime();
- purgeLibs(fs, LAUNCHER_PREFIX, current);
- purgeLibs(fs, SHARED_LIB_PREFIX, current);
+ Runnable purgeLibsRunnable = new Runnable() {
+ @Override
+ public void run() {
+ System.out.flush();
+ try {
+ //Only one server should purge sharelib
+ if (Services.get().get(JobsConcurrencyService.class).isFirstServer()) {
+ final Date current = Calendar.getInstance(TimeZone.getTimeZone("GMT")).getTime();
+ purgeLibs(fs, LAUNCHER_PREFIX, current);
+ purgeLibs(fs, SHARED_LIB_PREFIX, current);
+ }
+ }
+ catch (IOException e) {
+ LOG.error("There was an issue purging the sharelib", e);
+ }
}
- }
- catch (Exception e) {
- LOG.error("There was an issue purging the sharelib", e);
- }
-
+ };
+ services.get(SchedulerService.class).schedule(purgeLibsRunnable, 10,
+ services.getConf().getInt(PURGE_INTERVAL, 1) * 60 * 60 * 24, SchedulerService.Unit.SEC);
}
/**
@@ -369,15 +378,14 @@ public class ShareLibService implements Service, Instrumentable {
* @param fs the fs
* @param prefix the prefix
* @throws IOException Signals that an I/O exception has occurred.
- * @throws ParseException the parse exception
*/
- private void purgeLibs(FileSystem fs, final String prefix, final Date current) throws IOException, ParseException {
+ private void purgeLibs(FileSystem fs, final String prefix, final Date current) throws IOException {
Path executorLibBasePath = services.get(WorkflowAppService.class).getSystemLibPath();
PathFilter directoryFilter = new PathFilter() {
@Override
public boolean accept(Path path) {
if (path.getName().startsWith(prefix)) {
- String name = path.getName().toString();
+ String name = path.getName();
String time = name.substring(prefix.length());
Date d = null;
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 9788daf..63a91fa 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -2048,6 +2048,14 @@
</property>
<property>
+ <name>oozie.service.ShareLibService.purge.interval</name>
+ <value>1</value>
+ <description>
+ How often, in days, Oozie should check for old ShareLibs and LauncherLibs to purge from HDFS.
+ </description>
+ </property>
+
+ <property>
<name>oozie.action.jobinfo.enable</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index e1c5c5f..5fcbbda 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -106,16 +106,17 @@ public class TestShareLibService extends XFsTestCase {
setSystemProps();
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
- services.init();
- ShareLibService shareLibService = Services.get().get(ShareLibService.class);
- List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
- assertNotNull(launcherPath);
- assertTrue(getFileSystem().exists(launcherPath.get(0)));
- List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig");
- assertTrue(getFileSystem().exists(pigLauncherPath.get(0)));
-
- services.destroy();
-
+ try {
+ services.init();
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+ List<Path> launcherPath = shareLibService.getSystemLibJars(JavaActionExecutor.OOZIE_COMMON_LIBDIR);
+ assertNotNull(launcherPath);
+ assertTrue(getFileSystem().exists(launcherPath.get(0)));
+ List<Path> pigLauncherPath = shareLibService.getSystemLibJars("pig");
+ assertTrue(getFileSystem().exists(pigLauncherPath.get(0)));
+ } finally {
+ services.destroy();
+ }
}
@Test
@@ -125,33 +126,36 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
- services.init();
- String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
- + getNameNodeUri() + "</name-node>" + "</java>";
- Element eActionXml = XmlUtils.parseXml(actionXml);
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
- WorkflowJobBean wfj = new WorkflowJobBean();
- wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
- wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
- Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
- PigActionExecutor ae = new PigActionExecutor();
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
-
- URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
- String cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains(MyPig.class.getName() + ".jar"));
- assertTrue(cacheFilesStr.contains(MyOozie.class.getName() + ".jar"));
- // Hadoop 2 has two extra jars
- if (cacheFiles.length == 4) {
- assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
- assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
- }
- else {
- assertEquals(2, cacheFiles.length);
+ try {
+ services.init();
+ String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ + getNameNodeUri() + "</name-node>" + "</java>";
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ WorkflowJobBean wfj = new WorkflowJobBean();
+ wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+ wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
+ Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+ PigActionExecutor ae = new PigActionExecutor();
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+
+ URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ String cacheFilesStr = Arrays.toString(cacheFiles);
+ assertTrue(cacheFilesStr.contains(MyPig.class.getName() + ".jar"));
+ assertTrue(cacheFilesStr.contains(MyOozie.class.getName() + ".jar"));
+ // Hadoop 2 has two extra jars
+ if (cacheFiles.length == 4) {
+ assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
+ assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
+ }
+ else {
+ assertEquals(2, cacheFiles.length);
+ }
+ } finally {
+ services.destroy();
}
- services.destroy();
}
@Test
@@ -160,34 +164,36 @@ public class TestShareLibService extends XFsTestCase {
setSystemProps();
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
- services.init();
- String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
- + getNameNodeUri() + "</name-node>" + "</pig>";
- Element eActionXml = XmlUtils.parseXml(actionXml);
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
- WorkflowJobBean wfj = new WorkflowJobBean();
- wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
- wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
- Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
- PigActionExecutor ae = new PigActionExecutor();
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
-
- URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
- String cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains("MyPig.jar"));
- assertTrue(cacheFilesStr.contains("MyOozie.jar"));
- // Hadoop 2 has two extra jars
- if (cacheFiles.length == 4) {
- assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
- assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
- }
- else {
- assertEquals(2, cacheFiles.length);
+ try {
+ services.init();
+ String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ + getNameNodeUri() + "</name-node>" + "</pig>";
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ WorkflowJobBean wfj = new WorkflowJobBean();
+ wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+ wfj.setConf(XmlUtils.prettyPrint(new XConfiguration()).toString());
+ Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+ PigActionExecutor ae = new PigActionExecutor();
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+
+ URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ String cacheFilesStr = Arrays.toString(cacheFiles);
+ assertTrue(cacheFilesStr.contains("MyPig.jar"));
+ assertTrue(cacheFilesStr.contains("MyOozie.jar"));
+ // Hadoop 2 has two extra jars
+ if (cacheFiles.length == 4) {
+ assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
+ assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
+ }
+ else {
+ assertEquals(2, cacheFiles.length);
+ }
+ } finally {
+ services.destroy();
}
- services.destroy();
-
}
@Test
@@ -213,39 +219,41 @@ public class TestShareLibService extends XFsTestCase {
createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
- services.init();
- String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
- + getNameNodeUri() + "</name-node>"
- + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
- Element eActionXml = XmlUtils.parseXml(actionXml);
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
- WorkflowJobBean wfj = new WorkflowJobBean();
- protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
- wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
- wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
-
- Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
- PigActionExecutor ae = new PigActionExecutor();
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
- ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
-
- URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
- String cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains("MyPig.jar"));
- assertTrue(cacheFilesStr.contains("MyOozie.jar"));
- assertTrue(cacheFilesStr.contains("pig-10.jar"));
- // Hadoop 2 has two extra jars
- if (cacheFiles.length == 5) {
- assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
- assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
- }
- else {
- assertEquals(3, cacheFiles.length);
+ try {
+ services.init();
+ String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ + getNameNodeUri() + "</name-node>"
+ + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ WorkflowJobBean wfj = new WorkflowJobBean();
+ protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+ wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+ wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
+
+ Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+ PigActionExecutor ae = new PigActionExecutor();
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+
+ URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ String cacheFilesStr = Arrays.toString(cacheFiles);
+ assertTrue(cacheFilesStr.contains("MyPig.jar"));
+ assertTrue(cacheFilesStr.contains("MyOozie.jar"));
+ assertTrue(cacheFilesStr.contains("pig-10.jar"));
+ // Hadoop 2 has two extra jars
+ if (cacheFiles.length == 5) {
+ assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
+ assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
+ }
+ else {
+ assertEquals(3, cacheFiles.length);
+ }
+ } finally {
+ services.destroy();
}
- services.destroy();
-
}
@Test
@@ -310,12 +318,15 @@ public class TestShareLibService extends XFsTestCase {
Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1);
createDirs(fs, expirePath, noexpirePath, noexpirePath1);
- services.init();
- assertEquals(4, fs.listStatus(basePath).length);
- assertTrue(fs.exists(noexpirePath));
- assertTrue(fs.exists(noexpirePath1));
- assertTrue(fs.exists(expirePath));
- services.destroy();
+ try {
+ services.init();
+ assertEquals(4, fs.listStatus(basePath).length);
+ assertTrue(fs.exists(noexpirePath));
+ assertTrue(fs.exists(noexpirePath1));
+ assertTrue(fs.exists(expirePath));
+ } finally {
+ services.destroy();
+ }
}
// Logic is to keep all share-lib between current timestamp and 7days old + 1 latest sharelib older than 7 days.
@@ -326,7 +337,7 @@ public class TestShareLibService extends XFsTestCase {
setSystemProps();
Configuration conf = services.getConf();
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
- FileSystem fs = getFileSystem();
+ final FileSystem fs = getFileSystem();
// for directory created 8 days back to be deleted
long expiryTime = System.currentTimeMillis()
- TimeUnit.MILLISECONDS.convert(
@@ -336,7 +347,7 @@ public class TestShareLibService extends XFsTestCase {
String expireTs1 = dt.format(new Date(expiryTime - TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
String noexpireTs = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)));
String noexpireTs1 = dt.format(new Date(expiryTime + TimeUnit.MILLISECONDS.convert(2, TimeUnit.DAYS)));
- Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
+ final Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
Path expirePath = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs);
Path expirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + expireTs1);
@@ -344,14 +355,23 @@ public class TestShareLibService extends XFsTestCase {
Path noexpirePath1 = new Path(basePath, ShareLibService.LAUNCHER_PREFIX + noexpireTs1);
createDirs(fs, expirePath, expirePath1, noexpirePath, noexpirePath1);
- services.init();
- assertEquals(4, fs.listStatus(basePath).length);
- assertTrue(fs.exists(noexpirePath));
- assertTrue(fs.exists(noexpirePath1));
- assertTrue(fs.exists(expirePath));
- assertTrue(!fs.exists(expirePath1));
-
- services.destroy();
+ try {
+ services.init();
+ // Wait for the scheduled purge runnable to complete
+ waitFor(20 * 1000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ return (fs.listStatus(basePath).length == 4);
+ }
+ });
+ assertEquals(4, fs.listStatus(basePath).length);
+ assertTrue(fs.exists(noexpirePath));
+ assertTrue(fs.exists(noexpirePath1));
+ assertTrue(fs.exists(expirePath));
+ assertFalse(fs.exists(expirePath1));
+ } finally {
+ services.destroy();
+ }
}
@Test
@@ -371,13 +391,16 @@ public class TestShareLibService extends XFsTestCase {
fs.mkdirs(pigPath);
fs.mkdirs(pigPath1);
fs.mkdirs(pigPath2);
- services.init();
- ShareLibService shareLibService = Services.get().get(ShareLibService.class);
- assertNotNull(shareLibService.getShareLibJars("pig"));
- assertNotNull(shareLibService.getShareLibJars("pig_9"));
- assertNotNull(shareLibService.getShareLibJars("pig_10"));
- assertNull(shareLibService.getShareLibJars("pig_11"));
- services.destroy();
+ try {
+ services.init();
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+ assertNotNull(shareLibService.getShareLibJars("pig"));
+ assertNotNull(shareLibService.getShareLibJars("pig_9"));
+ assertNotNull(shareLibService.getShareLibJars("pig_10"));
+ assertNull(shareLibService.getShareLibJars("pig_11"));
+ } finally {
+ services.destroy();
+ }
}
@Test
@@ -396,15 +419,17 @@ public class TestShareLibService extends XFsTestCase {
Path path3 = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + dir3);
createDirs(fs, path1, path2, path3);
createFile(path1.toString() + Path.SEPARATOR + "pig" + Path.SEPARATOR + "pig.jar");
- services.init();
- ShareLibService shareLibService = Services.get().get(ShareLibService.class);
- assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
- services.destroy();
+ try {
+ services.init();
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+ assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
+ } finally {
+ services.destroy();
+ }
}
@Test
public void testShareLibLoadFile() throws Exception {
-
services = new Services();
FileSystem fs = getFileSystem();
setSystemProps();
@@ -412,68 +437,73 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
- services.init();
- ShareLibService shareLibService = Services.get().get(ShareLibService.class);
- assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
- assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
- fs.delete(new Path("shareLibPath/"), true);
- services.destroy();
+ try {
+ services.init();
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+ assertTrue(shareLibService.getShareLibJars("something_new").get(0).getName().endsWith("somethingNew.jar"));
+ assertTrue(shareLibService.getShareLibJars("pig").get(0).getName().endsWith("pig.jar"));
+ fs.delete(new Path("shareLibPath/"), true);
+ } finally {
+ services.destroy();
+ }
}
@Test
public void testLoadfromDFS() throws Exception {
services = new Services();
setSystemProps();
- services.init();
- FileSystem fs = getFileSystem();
- Date time = new Date(System.currentTimeMillis());
-
- Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
- Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time));
- fs.mkdirs(libpath);
-
- Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
- Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
- Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
- Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
- fs.mkdirs(pigPath);
- fs.mkdirs(ooziePath);
- fs.mkdirs(pigPath1);
- fs.mkdirs(pigPath2);
-
- createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
- createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
-
- String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
- + getNameNodeUri() + "</name-node>"
- + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
- Element eActionXml = XmlUtils.parseXml(actionXml);
- XConfiguration protoConf = new XConfiguration();
- protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
- WorkflowJobBean wfj = new WorkflowJobBean();
- protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
- wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
- wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
-
- Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
- PigActionExecutor ae = new PigActionExecutor();
- Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
- jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
- ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
- URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
- String cacheFilesStr = Arrays.toString(cacheFiles);
- assertTrue(cacheFilesStr.contains("pig-10.jar"));
- assertTrue(cacheFilesStr.contains("oozie_luncher.jar"));
- // Hadoop 2 has two extra jars
- if (cacheFiles.length == 4) {
- assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
- assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
- }
- else {
- assertEquals(2, cacheFiles.length);
+ try {
+ services.init();
+ FileSystem fs = getFileSystem();
+ Date time = new Date(System.currentTimeMillis());
+
+ Path basePath = new Path(services.getConf().get(WorkflowAppService.SYSTEM_LIB_PATH));
+ Path libpath = new Path(basePath, ShareLibService.SHARED_LIB_PREFIX + ShareLibService.dateFormat.format(time));
+ fs.mkdirs(libpath);
+
+ Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
+ Path ooziePath = new Path(libpath.toString() + Path.SEPARATOR + "oozie");
+ Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
+ Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
+ fs.mkdirs(pigPath);
+ fs.mkdirs(ooziePath);
+ fs.mkdirs(pigPath1);
+ fs.mkdirs(pigPath2);
+
+ createFile(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
+ createFile(libpath.toString() + Path.SEPARATOR + "oozie" + Path.SEPARATOR + "oozie_luncher.jar");
+
+ String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>"
+ + getNameNodeUri() + "</name-node>"
+ + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
+ Element eActionXml = XmlUtils.parseXml(actionXml);
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ WorkflowJobBean wfj = new WorkflowJobBean();
+ protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
+ wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
+ wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
+
+ Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
+ PigActionExecutor ae = new PigActionExecutor();
+ Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
+ jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
+ ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
+ URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
+ String cacheFilesStr = Arrays.toString(cacheFiles);
+ assertTrue(cacheFilesStr.contains("pig-10.jar"));
+ assertTrue(cacheFilesStr.contains("oozie_luncher.jar"));
+ // Hadoop 2 has two extra jars
+ if (cacheFiles.length == 4) {
+ assertTrue(cacheFilesStr.contains("MRAppJar.jar"));
+ assertTrue(cacheFilesStr.contains("hadoop-mapreduce-client-jobclient-"));
+ }
+ else {
+ assertEquals(2, cacheFiles.length);
+ }
+ } finally {
+ services.destroy();
}
- services.destroy();
-
}
@Test
@@ -485,12 +515,15 @@ public class TestShareLibService extends XFsTestCase {
Configuration conf = services.getConf();
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, fs.getUri() + "/user/test/config.properties");
conf.set(ShareLibService.SHIP_LAUNCHER_JAR, "true");
- services.init();
- ShareLibService shareLibService = Services.get().get(ShareLibService.class);
- assertNull(shareLibService.getShareLibJars("something_new"));
- assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
- fs.delete(new Path("shareLibPath/"), true);
- services.destroy();
+ try {
+ services.init();
+ ShareLibService shareLibService = Services.get().get(ShareLibService.class);
+ assertNull(shareLibService.getShareLibJars("something_new"));
+ assertEquals(shareLibService.getShareLibJars("pig").size(), 2);
+ fs.delete(new Path("shareLibPath/"), true);
+ } finally {
+ services.destroy();
+ }
}
public void createFile(String filename) throws IOException {
http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 8940746..81e26ef 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1783 Sharelib purging only occurs at Oozie startup (rkanter)
OOZIE-1689 HA support for OOZIE-7(Ability to view the log information corresponding to particular coordinator action) (puru via mona)
OOZIE-1849 If the underlying job finishes while a Workflow is suspended, Oozie can take a while to realize it (rkanter)
OOZIE-1835 NullPointerException from SLAEmailEventListener (rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/b9c3e0b0/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
index b2588b8..4c095ef 100644
--- a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
+++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
@@ -65,6 +65,9 @@ public class TestOozieSharelibCLI extends XTestCase {
@AfterClass
protected void tearDown() throws Exception {
System.setSecurityManager(SECURITY_MANAGER);
+ if (services != null) {
+ services.destroy();
+ }
super.tearDown();
}
@@ -160,7 +163,9 @@ public class TestOozieSharelibCLI extends XTestCase {
services = new Services();
services.getConf()
.set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService,"
- + "org.apache.oozie.service.HadoopAccessorService,org.apache.oozie.service.ShareLibService");
+ + "org.apache.oozie.service.SchedulerService,"
+ + "org.apache.oozie.service.HadoopAccessorService,"
+ + "org.apache.oozie.service.ShareLibService");
services.init();
}
return services;